def _recoverExistingVms(self):
        start_time = utils.monotonic_time()
        try:
            self.log.debug('recovery: started')

            # Starting up libvirt might take long when host under high load,
            # we prefer running this code in external thread to avoid blocking
            # API response.
            mog = min(config.getint('vars', 'max_outgoing_migrations'),
                      numa.cpu_topology().cores)
            migration.SourceThread.setMaxOutgoingMigrations(mog)

            recovery.all_vms(self)

            # recover stage 3: waiting for domains to go up
            self._waitForDomainsUp()

            recovery.clean_vm_files(self)

            self._recovery = False

            # Now if we have VMs to restore we should wait pool connection
            # and then prepare all volumes.
            # Actually, we need it just to get the resources for future
            # volumes manipulations
            self._waitForStoragePool()

            self._preparePathsForRecoveredVMs()

            self.log.info('recovery: completed in %is',
                          utils.monotonic_time() - start_time)

        except:
            self.log.exception("recovery: failed")
            raise

def wait_for_removal(path, timeout, wait=0.1):
    deadline = utils.monotonic_time() + timeout
    while True:
        if not os.path.exists(path):
            return True
        if utils.monotonic_time() > deadline:
            return False
        time.sleep(wait)

 def assertMaxDuration(self, maxtime):
     start = utils.monotonic_time()
     try:
         yield
     finally:
         elapsed = utils.monotonic_time() - start
         if maxtime < elapsed:
             self.fail("Operation was too slow %.2fs > %.2fs" %
                       (elapsed, maxtime))

 def send(self):
     disp = self._disp
     timeout = self._timeout
     duration = 0
     s = monotonic_time()
     while ((timeout is None or duration < timeout) and
            (disp.writable() or not self._connected.isSet())):
             td = timeout - duration if timeout is not None else None
             self.process(td)
             duration = monotonic_time() - s

 def _work():
     invokations[0] += 1
     invokations[1] = monotonic_time()
     if invokations[0] == BLOCK_AT:
         # must be > (PERIOD * TIMES) ~= forever
         time.sleep(10 * PERIOD * TIMES)
     executions[0] += 1
     executions[1] = monotonic_time()
     if invokations[0] == TIMES:
         done.set()

    def test_timeout_not_triggered(self):
        time_start = monotonic_time()
        with monitor.Monitor(timeout=self.TIMEOUT) as mon:
            dummy_name = dummy.create()
            dummy.remove(dummy_name)

            for event in mon:
                break

        self.assertTrue((monotonic_time() - time_start) <= self.TIMEOUT)
        self.assertTrue(mon.is_stopped())

 def _serveRequest(self, ctx, req):
     start_time = monotonic_time()
     response = self._handle_request(req, ctx.server_address)
     error = getattr(response, "error", None)
     if error is None:
         response_log = "succeeded"
     else:
         response_log = "failed (error %s)" % (error.code,)
     self.log.info("RPC call %s %s in %.2f seconds",
                   req.method, response_log, monotonic_time() - start_time)
     if response is not None:
         ctx.requestDone(response)

    def recv(self):
        timeout = self._timeout
        s = monotonic_time()
        duration = 0
        while timeout is None or duration < timeout:
            try:
                return self._inbox.popleft()
            except IndexError:
                td = timeout - duration if timeout is not None else None
                self.process(td)
                duration = monotonic_time() - s

        return None

def nic(name, model, mac, start_sample, end_sample, interval):
    ifSpeed = [100, 1000][model in ('e1000', 'virtio')]

    ifStats = {'macAddr': mac,
               'name': name,
               'speed': str(ifSpeed),
               'state': 'unknown'}

    ifStats['rxErrors'] = str(end_sample['rx.errs'])
    ifStats['rxDropped'] = str(end_sample['rx.drop'])
    ifStats['txErrors'] = str(end_sample['tx.errs'])
    ifStats['txDropped'] = str(end_sample['tx.drop'])

    rxDelta = (
        end_sample['rx.bytes'] - start_sample['rx.bytes'])
    ifRxBytes = (100.0 *
                 (rxDelta % 2 ** 32) /
                 interval / ifSpeed / _MBPS_TO_BPS)
    txDelta = (
        end_sample['tx.bytes'] - start_sample['tx.bytes'])
    ifTxBytes = (100.0 *
                 (txDelta % 2 ** 32) /
                 interval / ifSpeed / _MBPS_TO_BPS)

    ifStats['rxRate'] = '%.1f' % ifRxBytes
    ifStats['txRate'] = '%.1f' % ifTxBytes

    ifStats['rx'] = str(end_sample['rx.bytes'])
    ifStats['tx'] = str(end_sample['tx.bytes'])
    ifStats['sampleTime'] = monotonic_time()

    return ifStats

 def tick(self):
     now = monotonic_time()
     result = self._result(now)
     self._counter = (self._counter + 1) % self._interval
     if result:
         self._last_time = now
     return result

    def _scan(self):
        with closing(select.epoll()) as epoll:
            with _monitoring_socket(self._queue, self._groups, epoll) as sock:
                with _pipetrick(epoll) as self._pipetrick:
                    self._scanning_started.set()
                    while True:
                        if self._timeout:
                            timeout = self._end_time - monotonic_time()
                            # timeout expired
                            if timeout <= 0:
                                self._scanning_stopped.set()
                                self._queue.put(_TIMEOUT_FLAG)
                                break
                        else:
                            timeout = -1

                        events = NoIntrPoll(epoll.poll, timeout=timeout)
                        # poll timeouted
                        if len(events) == 0:
                            self._scanning_stopped.set()
                            self._queue.put(_TIMEOUT_FLAG)
                            break
                        # stopped by pipetrick
                        elif (self._pipetrick[0], select.POLLIN) in events:
                            NoIntrCall(os.read, self._pipetrick[0], 1)
                            self._queue.put(_STOP_FLAG)
                            break

                        _nl_recvmsgs_default(sock)

 def _attempt_log_stats(self):
     self._counter += 1
     if monotonic_time() > self._next_report:
         self.log.info('%s requests processed during %s seconds',
                       self._counter, self._timeout)
         self._next_report += self._timeout
         self._counter = 0

 def __init__(self, bridge, timeout, threadFactory=None):
     self._bridge = bridge
     self._workQueue = Queue()
     self._threadFactory = threadFactory
     self._timeout = timeout
     self._next_report = monotonic_time() + self._timeout
     self._counter = 0

 def __init__(self, sslctx, handshake_finished_handler, handshake_timeout=SSL_HANDSHAKE_TIMEOUT):
     self._give_up_at = monotonic_time() + handshake_timeout
     self._has_been_set_up = False
     self._is_handshaking = True
     self.want_read = True
     self.want_write = True
     self._sslctx = sslctx
     self._handshake_finished_handler = handshake_finished_handler

def _resize_map(name):
    """
    Invoke multipathd to resize a device
    Must run as root

    Raises Error if multipathd failed to resize the map.
    """
    log.debug("Resizing map %r", name)
    cmd = [_MULTIPATHD.cmd, "resize", "map", name]
    start = utils.monotonic_time()
    rc, out, err = utils.execCmd(cmd, raw=True, execCmdLogger=log)
    # multipathd reports some errors using non-zero exit code and stderr (need
    # to be root), but the command may return 0, and the result is reported
    # using stdout.
    if rc != 0 or out != "ok\n":
        raise Error("Resizing map %r failed: out=%r err=%r"
                    % (name, out, err))
    elapsed = utils.monotonic_time() - start
    log.debug("Resized map %r in %.2f seconds", name, elapsed)

    def testTimeoutNoTimeForSleep(self):
        # time  action
        # 0     first attempt
        # 1     bail out (1 + 1 == deadline)
        with FakeMonotonicTime(0):

            def operation():
                time.sleep(1)
                raise RuntimeError

            self.assertRaises(RuntimeError, utils.retry, operation,
                              timeout=2, sleep=1)
            self.assertEqual(utils.monotonic_time(), 1)

    def testTimeoutDeadlineReached(self):
        # time  action
        # 0     first attempt
        # 1     sleep
        # 2     second attempt
        # 3     bail out (3 == deadline)
        with FakeMonotonicTime(0):

            def operation():
                time.sleep(1)
                raise RuntimeError

            self.assertRaises(RuntimeError, utils.retry, operation,
                              timeout=3, sleep=1)
            self.assertEqual(utils.monotonic_time(), 3)

    def testTimeoutSleepOnce(self):
        # time  action
        # 0     first attempt
        # 2     sleep
        # 3     second attempt
        # 5     bail out (5 > deadline)
        with FakeMonotonicTime(0):
            counter = [0]

            def operation():
                time.sleep(2)
                counter[0] += 1
                raise RuntimeError

            self.assertRaises(RuntimeError, utils.retry, operation,
                              timeout=4, sleep=1)
            self.assertEqual(counter[0], 2)
            self.assertEqual(utils.monotonic_time(), 5)

def _nic_traffic(vm_obj, name, model, mac,
                 start_sample, start_index,
                 end_sample, end_index, interval):
    ifSpeed = [100, 1000][model in ('e1000', 'virtio')]

    ifStats = {'macAddr': mac,
               'name': name,
               'speed': str(ifSpeed),
               'state': 'unknown'}

    with _skip_if_missing_stats(vm_obj):
        ifStats['rxErrors'] = str(end_sample['net.%d.rx.errs' % end_index])
        ifStats['rxDropped'] = str(end_sample['net.%d.rx.drop' % end_index])
        ifStats['txErrors'] = str(end_sample['net.%d.tx.errs' % end_index])
        ifStats['txDropped'] = str(end_sample['net.%d.tx.drop' % end_index])

    with _skip_if_missing_stats(vm_obj):
        rxDelta = (
            end_sample['net.%d.rx.bytes' % end_index] -
            start_sample['net.%d.rx.bytes' % start_index]
        )
        txDelta = (
            end_sample['net.%d.tx.bytes' % end_index] -
            start_sample['net.%d.tx.bytes' % start_index]
        )

    ifRxBytes = (100.0 *
                 (rxDelta % 2 ** 32) /
                 interval / ifSpeed / _MBPS_TO_BPS)
    ifTxBytes = (100.0 *
                 (txDelta % 2 ** 32) /
                 interval / ifSpeed / _MBPS_TO_BPS)

    ifStats['rxRate'] = '%.1f' % ifRxBytes
    ifStats['txRate'] = '%.1f' % ifTxBytes

    ifStats['rx'] = str(end_sample['net.%d.rx.bytes' % end_index])
    ifStats['tx'] = str(end_sample['net.%d.tx.bytes' % end_index])
    ifStats['sampleTime'] = monotonic_time()

    return ifStats

    def test_stop(self):
        PERIOD = 0.1

        invocations = [0]

        def _work():
            invocations[0] = monotonic_time()

        op = periodic.Operation(_work, period=PERIOD, scheduler=self.sched, executor=self.exc)
        op.start()
        time.sleep(PERIOD * 2)
        # avoid pathological case on which nothing ever runs
        self.assertTrue(invocations[0] > 0)

        op.stop()

        # cooldown. Let's try to avoid scheduler mistakes.
        time.sleep(PERIOD)
        stop = monotonic_time()

        self.assertTrue(stop > invocations[0])