def ssh_execute(ssh, cmd, process_input=None,
                addl_env=None, check_exit_code=True):
    LOG.debug(_('Running cmd (SSH): %s'), cmd)
    if addl_env:
        raise InvalidArgumentError(_('Environment not supported over SSH'))

    if process_input:
        # This is (probably) fixable if we need it...
        raise InvalidArgumentError(_('process_input not supported over SSH'))

    stdin_stream, stdout_stream, stderr_stream = ssh.exec_command(cmd)
    channel = stdout_stream.channel

    # NOTE(justinsb): This seems suspicious...
    # ...other SSH clients have buffering issues with this approach
    stdout = stdout_stream.read()
    stderr = stderr_stream.read()
    stdin_stream.close()

    exit_status = channel.recv_exit_status()

    # exit_status == -1 if no exit code was returned
    if exit_status != -1:
        LOG.debug(_('Result was %s') % exit_status)
        if check_exit_code and exit_status != 0:
            raise ProcessExecutionError(exit_code=exit_status,
                                        stdout=stdout,
                                        stderr=stderr,
                                        cmd=cmd)

    return (stdout, stderr)

    def wait(self):
        """Loop waiting on children to die and respawning as necessary"""

        LOG.debug(_('Full set of CONF:'))
        CONF.log_opt_values(LOG, std_logging.DEBUG)

        while self.running:
            wrap = self._wait_child()
            if not wrap:
                continue

            while self.running and len(wrap.children) < wrap.workers:
                self._start_child(wrap)

        if self.sigcaught:
            signame = {signal.SIGTERM: 'SIGTERM',
                       signal.SIGINT: 'SIGINT'}[self.sigcaught]
            LOG.info(_('Caught %s, stopping children'), signame)

        for pid in self.children:
            try:
                os.kill(pid, signal.SIGTERM)
            except OSError as exc:
                if exc.errno != errno.ESRCH:
                    raise

        # Wait for children to die
        if self.children:
            LOG.info(_('Waiting on %d children to exit'), len(self.children))
            while self.children:
                self._wait_child()

    def _run_loop(self, run_once=False):
        next_run = None
        current_run = None

        while self.running:
            current_run = timeutils.isotime()
            next_run = time.time() + CONF.scheduler.job_schedule_interval

            # do work
            with utils.log_warning_and_dismiss_exception(LOG):
                self.enqueue_jobs(end_time=current_run)

            # if shutdown hasn't been requested, do nothing until next run
            if self.running:
                seconds = next_run - time.time()
                if seconds > 0:
                    time.sleep(seconds)
                else:
                    msg = _('Scheduling of jobs took longer than expected.')
                    LOG.warn(msg)

            if run_once:
                break

        LOG.info(_('Scheduler is shutting down'))

    def wait(self):
        """Loop waiting on children to die and respawning as necessary"""

        LOG.debug(_('Full set of CONF:'))
        CONF.log_opt_values(LOG, std_logging.DEBUG)

        while self.running:
            wrap = self._wait_child()
            if not wrap:
                # Yield to other threads if no children have exited
                # Sleep for a short time to avoid excessive CPU usage
                # (see bug #1095346)
                eventlet.greenthread.sleep(.01)
                continue

            while self.running and len(wrap.children) < wrap.workers:
                self._start_child(wrap)

        if self.sigcaught:
            signame = {signal.SIGTERM: 'SIGTERM',
                       signal.SIGINT: 'SIGINT'}[self.sigcaught]
            LOG.info(_('Caught %s, stopping children'), signame)

        for pid in self.children:
            try:
                os.kill(pid, signal.SIGTERM)
            except OSError as exc:
                if exc.errno != errno.ESRCH:
                    raise

        # Wait for children to die
        if self.children:
            LOG.info(_('Waiting on %d children to exit'), len(self.children))
            while self.children:
                self._wait_child()

    def reconnect(self):
        """Handles reconnecting and re-establishing sessions and queues"""
        if self.connection.opened():
            try:
                self.connection.close()
            except qpid_exceptions.ConnectionError:
                pass

        attempt = 0
        delay = 1
        while True:
            broker = self.brokers[attempt % len(self.brokers)]
            attempt += 1

            try:
                self.connection_create(broker)
                self.connection.open()
            except qpid_exceptions.ConnectionError, e:
                msg_dict = dict(e=e, delay=delay)
                msg = _("Unable to connect to AMQP server: %(e)s. "
                        "Sleeping %(delay)s seconds") % msg_dict
                LOG.error(msg)
                time.sleep(delay)
                delay = min(2 * delay, 60)
            else:
                LOG.info(_('Connected to AMQP server on %s'), broker)
                break

    def _process_retention(self, instance_id, schedule_id):
        LOG.debug(_("Processing retention."))
        retention = self._get_retention(instance_id)

        if retention > 0:
            scheduled_images = self._find_scheduled_images_for_server(
                instance_id)

            if len(scheduled_images) > retention:
                to_delete = scheduled_images[retention:]
                LOG.info(_('[%(worker_tag)s] Removing %(remove)d '
                           'images for a retention of %(retention)d') %
                         {'worker_tag': self.get_worker_tag(),
                          'remove': len(to_delete),
                          'retention': retention})
                for image in to_delete:
                    image_id = image.id
                    self._get_nova_client().images.delete(image_id)
                    LOG.info(_('[%(worker_tag)s] Removed image '
                               '%(image_id)s') %
                             {'worker_tag': self.get_worker_tag(),
                              'image_id': image_id})
        else:
            msg = ("[%(worker_tag)s] Retention %(retention)s found for "
                   "schedule %(schedule)s for %(instance)s"
                   % {'worker_tag': self.get_worker_tag(),
                      'retention': retention,
                      'schedule': schedule_id,
                      'instance': instance_id})
            LOG.info(msg)

def multicall(conf, context, topic, msg, timeout, connection_pool):
    """Make a call that returns multiple times."""
    # TODO(pekowski): Remove all these comments in Havana.
    # For amqp_rpc_single_reply_queue = False,
    # Can't use 'with' for multicall, as it returns an iterator
    # that will continue to use the connection.  When it's done,
    # connection.close() will get called which will put it back into
    # the pool
    # For amqp_rpc_single_reply_queue = True,
    # The 'with' statement is mandatory for closing the connection
    LOG.debug(_('Making synchronous call on %s ...'), topic)
    msg_id = uuid.uuid4().hex
    msg.update({'_msg_id': msg_id})
    LOG.debug(_('MSG_ID is %s') % (msg_id))
    _add_unique_id(msg)
    pack_context(msg, context)

    # TODO(pekowski): Remove this flag and the code under the if clause
    #                 in Havana.
    if not conf.amqp_rpc_single_reply_queue:
        conn = ConnectionContext(conf, connection_pool)
        wait_msg = MulticallWaiter(conf, conn, timeout)
        conn.declare_direct_consumer(msg_id, wait_msg)
        conn.topic_send(topic, rpc_common.serialize_msg(msg), timeout)
    else:
        with _reply_proxy_create_sem:
            if not connection_pool.reply_proxy:
                connection_pool.reply_proxy = ReplyProxy(conf, connection_pool)
        msg.update({'_reply_q': connection_pool.reply_proxy.get_reply_q()})
        wait_msg = MulticallProxyWaiter(conf, msg_id, timeout, connection_pool)
        with ConnectionContext(conf, connection_pool) as conn:
            conn.topic_send(topic, rpc_common.serialize_msg(msg), timeout)
    return wait_msg

    def consume_in_thread(self):
        """Runs the ZmqProxy service"""
        ipc_dir = CONF.rpc_zmq_ipc_dir
        consume_in = "tcp://%s:%s" % \
            (CONF.rpc_zmq_bind_address,
             CONF.rpc_zmq_port)
        consumption_proxy = InternalContext(None)

        if not os.path.isdir(ipc_dir):
            try:
                utils.execute('mkdir', '-p', ipc_dir, run_as_root=True)
                utils.execute('chown', "%s:%s" % (os.getuid(), os.getgid()),
                              ipc_dir, run_as_root=True)
                utils.execute('chmod', '750', ipc_dir, run_as_root=True)
            except utils.ProcessExecutionError:
                with excutils.save_and_reraise_exception():
                    LOG.error(_("Could not create IPC directory %s") %
                              (ipc_dir, ))

        try:
            self.register(consumption_proxy,
                          consume_in,
                          zmq.PULL,
                          out_bind=True)
        except zmq.ZMQError:
            with excutils.save_and_reraise_exception():
                LOG.error(_("Could not create ZeroMQ receiver daemon. "
                            "Socket may already be in use."))

        super(ZmqProxy, self).consume_in_thread()

    def __init__(self, addr, zmq_type, bind=True, subscribe=None):
        self.sock = _get_ctxt().socket(zmq_type)
        self.addr = addr
        self.type = zmq_type
        self.subscriptions = []

        # Support failures on sending/receiving on wrong socket type.
        self.can_recv = zmq_type in (zmq.PULL, zmq.SUB)
        self.can_send = zmq_type in (zmq.PUSH, zmq.PUB)
        self.can_sub = zmq_type in (zmq.SUB, )

        # Support list, str, & None for subscribe arg (cast to list)
        do_sub = {
            list: subscribe,
            str: [subscribe],
            type(None): []
        }[type(subscribe)]

        for f in do_sub:
            self.subscribe(f)

        str_data = {'addr': addr, 'type': self.socket_s(),
                    'subscribe': subscribe, 'bind': bind}

        LOG.debug(_("Connecting to %(addr)s with %(type)s"), str_data)
        LOG.debug(_("-> Subscribed to %(subscribe)s"), str_data)
        LOG.debug(_("-> bind: %(bind)s"), str_data)

        try:
            if bind:
                self.sock.bind(addr)
            else:
                self.sock.connect(addr)
        except Exception:
            raise RPCException(_("Could not open socket."))

    def register(self, proxy, in_addr, zmq_type_in, out_addr=None,
                 zmq_type_out=None, in_bind=True, out_bind=True,
                 subscribe=None):

        LOG.info(_("Registering reactor"))

        if zmq_type_in not in (zmq.PULL, zmq.SUB):
            raise RPCException("Bad input socktype")

        # Items push in.
        inq = ZmqSocket(in_addr, zmq_type_in, bind=in_bind,
                        subscribe=subscribe)

        self.proxies[inq] = proxy
        self.sockets.append(inq)

        LOG.info(_("In reactor registered"))

        if not out_addr:
            return

        if zmq_type_out not in (zmq.PUSH, zmq.PUB):
            raise RPCException("Bad output socktype")

        # Items push out.
        outq = ZmqSocket(out_addr, zmq_type_out, bind=out_bind)

        self.mapping[inq] = outq
        self.mapping[outq] = inq
        self.sockets.append(outq)

        LOG.info(_("Out reactor registered"))

    def _run_loop(self, run_once=False, poll_once=False):
        self.running = True
        self.pid = os.getpid()
        self.processor.init_processor(self)
        self.worker_id = self._register_worker()

        while self.running:
            job = self._poll_for_next_job(poll_once)
            if job:
                LOG.debug(_('Processing job: %s') % job)
                try:
                    self.processor.process_job(job)
                except Exception as e:
                    msg = _("Worker %(worker_id)s Error processing job:"
                            " %(job)s")
                    LOG.exception(msg % {'worker_id': self.worker_id,
                                         'job': job['id']})
                    self.update_job(job['id'], 'ERROR',
                                    error_message=unicode(e))

            if run_once:
                self.running = False

        LOG.info(_('Worker is shutting down'))
        self._unregister_worker()
        self.processor.cleanup_processor()

    def _run_loop(self, run_once=False, poll_once=False):
        self.running = True
        self.pid = os.getpid()
        self.processor.init_processor(self)
        self.worker_id = self._register_worker()

        while self.running:
            time_before = time.time()
            job = self._poll_for_next_job(poll_once)
            if job:
                LOG.debug(_('Processing job: %s') % job)
                try:
                    self.processor.process_job(job)
                except Exception as e:
                    msg = _("Worker %(worker_id)s Error processing job:"
                            " %(job)s")
                    LOG.exception(msg % {'worker_id': self.worker_id,
                                         'job': job['id']})
                    self.update_job(job['id'], 'ERROR',
                                    error_message=unicode(e))

                time_after = time.time()

                # Ensure that we wait at least job_poll_interval between jobs
                time_delta = time_after - time_before
                if time_delta < CONF.worker.job_poll_interval:
                    time.sleep(CONF.worker.job_poll_interval - time_delta)

            if run_once:
                self.running = False

        LOG.info(_('Worker is shutting down'))
        self._unregister_worker()
        self.processor.cleanup_processor()

    def _poll_image_status(self, job, image_id):
        try:
            image_status = self._get_image_status(image_id)
        except Exception:
            exc_type, exc_value, exc_tb = sys.exc_info()
            org_err_msg = tb.format_exception_only(exc_type, exc_value)
            err_val = {'job_id': job['id'],
                       'image_id': image_id,
                       'org_err_msg': org_err_msg}
            err_msg = (
                _("PollingExc: image: %(image_id)s, err: %(org_err_msg)s") %
                err_val)
            LOG.exception('[%(worker_tag)s] %(msg)s'
                          % {'worker_tag': self.get_worker_tag(),
                             'msg': err_msg})
            self._job_error_occurred(job, error_message=err_msg)
            raise exc.PollingException(err_msg)

        if image_status is None or image_status in _FAILED_IMAGE_STATUSES:
            err_val = {'image_id': image_id,
                       "image_status": image_status,
                       "job_id": job['id']}
            err_msg = (
                _("PollingErr: Got failed image status. Details:"
                  " image_id: %(image_id)s, 'image_status': %(image_status)s"
                  " job_id: %(job_id)s") % err_val)
            self._job_error_occurred(job, error_message=err_msg)
            raise exc.PollingException(err_msg)
        return image_status

    def create_consumer(self, topic, proxy, fanout=False):
        # Register with matchmaker.
        _get_matchmaker().register(topic, CONF.rpc_zmq_host)

        # Subscription scenarios
        if fanout:
            sock_type = zmq.SUB
            subscribe = ('', fanout)[type(fanout) == str]
            topic = 'fanout~' + topic.split('.', 1)[0]
        else:
            sock_type = zmq.PULL
            subscribe = None
            topic = '.'.join((topic.split('.', 1)[0], CONF.rpc_zmq_host))

        if topic in self.topics:
            LOG.info(_("Skipping topic registration. Already registered."))
            return

        # Receive messages from (local) proxy
        inaddr = "ipc://%s/zmq_topic_%s" % \
            (CONF.rpc_zmq_ipc_dir, topic)

        LOG.debug(_("Consumer is a zmq.%s"),
                  ['PULL', 'SUB'][sock_type == zmq.SUB])

        self.reactor.register(proxy, inaddr, sock_type,
                              subscribe=subscribe, in_bind=False)
        self.topics.append(topic)

def _multi_send(method, context, topic, msg, timeout=None,
                envelope=False, _msg_id=None):
    """
    Wraps the sending of messages,
    dispatches to the matchmaker and sends
    message to all relevant hosts.
    """
    conf = CONF
    LOG.debug(_("%(msg)s") % {'msg': ' '.join(map(pformat, (topic, msg)))})

    queues = _get_matchmaker().queues(topic)
    LOG.debug(_("Sending message(s) to: %s"), queues)

    # Don't stack if we have no matchmaker results
    if len(queues) == 0:
        LOG.warn(_("No matchmaker results. Not casting."))
        # While not strictly a timeout, callers know how to handle
        # this exception and a timeout isn't too big a lie.
        raise rpc_common.Timeout(_("No match from matchmaker."))

    # This supports brokerless fanout (addresses > 1)
    for queue in queues:
        (_topic, ip_addr) = queue
        _addr = "tcp://%s:%s" % (ip_addr, conf.rpc_zmq_port)

        if method.__name__ == '_cast':
            eventlet.spawn_n(method, _addr, context,
                             _topic, msg, timeout, envelope,
                             _msg_id)
            return
        return method(_addr, context, _topic, msg, timeout,
                      envelope)

    def process_request(self, req):
        """Try to find a version first in the accept header, then the URL."""
        msg = _("Determining version of request: %(method)s %(path)s"
                " Accept: %(accept)s")
        args = {'method': req.method, 'path': req.path, 'accept': req.accept}
        LOG.debug(msg % args)

        accept = str(req.accept)
        if accept.startswith('application/vnd.openstack.qonos-'):
            LOG.debug(_("Using media-type versioning"))
            token_loc = len('application/vnd.openstack.qonos-')
            req_version = accept[token_loc:]
        else:
            LOG.debug(_("Using url versioning"))
            # Remove version in url so it doesn't conflict later
            req_version = req.path_info_pop()
        try:
            version = self._match_version_string(req_version)
        except ValueError:
            LOG.debug(_("Unknown version. Returning version choices."))
            return self.versions_app

        req.environ['api.version'] = version
        req.path_info = ''.join(('/v', str(version), req.path_info or '/'))
        LOG.debug(_("Matched version: v%d"), version)
        LOG.debug('new uri %s' % req.path_info)
        return None

    def _process_data(self, ctxt, version, method, args):
        """Process a message in a new thread.

        If the proxy object we have has a dispatch method
        (see rpc.dispatcher.RpcDispatcher), pass it the version,
        method, and args and let it dispatch as appropriate.  If not, use
        the old behavior of magically calling the specified method on the
        proxy we have here.
        """
        ctxt.update_store()
        try:
            rval = self.proxy.dispatch(ctxt, version, method, **args)
            # Check if the result was a generator
            if inspect.isgenerator(rval):
                for x in rval:
                    ctxt.reply(x, None, connection_pool=self.connection_pool)
            else:
                ctxt.reply(rval, None, connection_pool=self.connection_pool)
            # This final None tells multicall that it is done.
            ctxt.reply(ending=True, connection_pool=self.connection_pool)
        except rpc_common.ClientException as e:
            LOG.debug(_('Expected exception during message handling (%s)') %
                      e._exc_info[1])
            ctxt.reply(None, e._exc_info,
                       connection_pool=self.connection_pool,
                       log_failure=False)
        except Exception:
            # sys.exc_info() is deleted by LOG.exception().
            exc_info = sys.exc_info()
            LOG.error(_('Exception during message handling'),
                      exc_info=exc_info)
            ctxt.reply(None, exc_info, connection_pool=self.connection_pool)

    def consume(self, sock):
        #TODO(ewindisch): use zero-copy (i.e. references, not copying)
        data = sock.recv()
        LOG.debug(_("CONSUMER RECEIVED DATA: %s"), data)
        if sock in self.mapping:
            LOG.debug(_("ROUTER RELAY-OUT %(data)s") % {
                'data': data})
            self.mapping[sock].send(data)
            return

        proxy = self.proxies[sock]

        if data[2] == 'cast':  # Legacy protocol
            packenv = data[3]

            ctx, msg = _deserialize(packenv)
            request = rpc_common.deserialize_msg(msg)
            ctx = RpcContext.unmarshal(ctx)
        elif data[2] == 'impl_zmq_v2':
            packenv = data[4:]

            msg = unflatten_envelope(packenv)
            request = rpc_common.deserialize_msg(msg)

            # Unmarshal only after verifying the message.
            ctx = RpcContext.unmarshal(data[3])
        else:
            LOG.error(_("ZMQ Envelope version unsupported or unknown."))
            return

        self.pool.spawn_n(self.process, proxy, ctx, request)

    def __call__(self, message_data):
        """Consumer callback to call a method on a proxy object.

        Parses the message for validity and fires off a thread to call the
        proxy object method.

        Message data should be a dictionary with two keys:
            method: string representing the method to call
            args: dictionary of arg: value

        Example: {'method': 'echo', 'args': {'value': 42}}

        """
        # It is important to clear the context here, because at this point
        # the previous context is stored in local.store.context
        if hasattr(local.store, 'context'):
            del local.store.context
        rpc_common._safe_log(LOG.debug, _('received %s'), message_data)
        self.msg_id_cache.check_duplicate_message(message_data)
        ctxt = unpack_context(self.conf, message_data)
        method = message_data.get('method')
        args = message_data.get('args', {})
        version = message_data.get('version', None)
        if not method:
            LOG.warn(_('no method for message: %s') % message_data)
            ctxt.reply(_('No method for message: %s') % message_data,
                       connection_pool=self.connection_pool)
            return
        self.pool.spawn_n(self._process_data, ctxt, version, method, args)

    def _process_retention(self, instance_id, schedule_id):
        LOG.debug(_("Processing retention."))
        retention = self._get_retention(instance_id)

        if retention > 0:
            scheduled_images = self._find_scheduled_images_for_server(
                instance_id)

            if len(scheduled_images) > retention:
                to_delete = scheduled_images[retention:]
                LOG.info(_('Worker %(worker_id)s '
                           'Removing %(remove)d images for a retention '
                           'of %(retention)d')
                          % {'worker_id': self.worker.worker_id,
                             'remove': len(to_delete),
                             'retention': retention})
                for image in to_delete:
                    image_id = image.id
                    self._get_nova_client().images.delete(image_id)
                    LOG.info(_('Worker %(worker_id)s Removed image '
                               '%(image_id)s')
                              % {'worker_id': self.worker.worker_id,
                                 'image_id': image_id})
        else:
            self._delete_schedule(schedule_id, instance_id)