def consume_in_thread(self):
        """Runs the ZmqProxy service"""
        ipc_dir = CONF.rpc_zmq_ipc_dir
        consume_in = "tcp://%s:%s" % \
            (CONF.rpc_zmq_bind_address,
             CONF.rpc_zmq_port)
        consumption_proxy = InternalContext(None)

        if not os.path.isdir(ipc_dir):
            try:
                utils.execute('mkdir', '-p', ipc_dir, run_as_root=True)
                utils.execute('chown', "%s:%s" % (os.getuid(), os.getgid()),
                              ipc_dir, run_as_root=True)
                utils.execute('chmod', '750', ipc_dir, run_as_root=True)
            except utils.ProcessExecutionError:
                with excutils.save_and_reraise_exception():
                    LOG.error(_("Could not create IPC directory %s") %
                              (ipc_dir, ))

        try:
            self.register(consumption_proxy,
                          consume_in,
                          zmq.PULL,
                          out_bind=True)
        except zmq.ZMQError:
            with excutils.save_and_reraise_exception():
                LOG.error(_("Could not create ZeroMQ receiver daemon. "
                            "Socket may already be in use."))

        super(ZmqProxy, self).consume_in_thread()

    def update(self, req, id, body):
        LOG.debug(_('Updating app node by id %s' % id))
        context = req.environ['vsm.context']

        if not self.is_valid_body(body, 'appnode'):
            raise exc.HTTPBadRequest()

        body = body.get('appnode')
        LOG.debug('PUT body: %s' % body)

        if id is None:
            expl = _('Request body and URI mismatch: No appnode_id')
            raise webob.exc.HTTPBadRequest(explanation=expl)
        # convert from unicode to str
        id = str(id)
        status = body.get('ssh_status', None)
        log = body.get('log_info', None)
        ip = body.get('ip', None)

        if not status and not log and not ip:
            expl = _('Request body and URI mismatch: ssh_status or log_info required.')
            raise webob.exc.HTTPBadRequest(explanation=expl)

        appnodes.update(context, id, status, log, ip)
        return webob.Response(status_int=201)

    def reconnect(self):
        """Handles reconnecting and re-establishing sessions and queues"""
        if self.connection.opened():
            try:
                self.connection.close()
            except qpid_exceptions.ConnectionError:
                pass

        attempt = 0
        delay = 1
        while True:
            broker = self.brokers[attempt % len(self.brokers)]
            attempt += 1

            try:
                self.connection_create(broker)
                self.connection.open()
            except qpid_exceptions.ConnectionError, e:
                msg_dict = dict(e=e, delay=delay)
                msg = _("Unable to connect to AMQP server: %(e)s. "
                        "Sleeping %(delay)s seconds") % msg_dict
                LOG.error(msg)
                time.sleep(delay)
                delay = min(2 * delay, 60)
            else:
                LOG.info(_('Connected to AMQP server on %s'), broker)
                break

def load_physical_driver(physical_driver=None):
    """Load ad physical driver module.

    Load the physical driver module specified by the physical_driver
    configuration option or, if supplied, the driver name supplied
    as an argument.

    :param physical_driver: a physical driver name to override the config opt.
    :returns: a PhysicalDriver interface.
    """
    if not physical_driver:
        physical_driver = CONF.physical_driver

    if not physical_driver:
        LOG.warn(_("Physical driver option required, but not specified"))
        raise

    try:
        # If just write driver.CephDriver
        driver = importutils.import_object_ns("vsm.physical", physical_driver)
        return utils.check_isinstance(driver, PhysicalDriver)
    except ImportError:
        try:
            # If vsm.physical.driver.CephDriver
            driver = importutils.import_object(physical_driver)
            return utils.check_isinstance(driver, PhysicalDriver)
        except ImportError:
            LOG.exception(_("Unable to load the physical driver"))
            raise

    def consume(self, sock):
        #TODO(ewindisch): use zero-copy (i.e. references, not copying)
        data = sock.recv()
        LOG.debug(_("CONSUMER RECEIVED DATA: %s"), data)
        if sock in self.mapping:
            LOG.debug(_("ROUTER RELAY-OUT %(data)s") % {
                'data': data})
            self.mapping[sock].send(data)
            return

        proxy = self.proxies[sock]

        if data[2] == 'cast':  # Legacy protocol
            packenv = data[3]

            ctx, msg = _deserialize(packenv)
            request = rpc_common.deserialize_msg(msg)
            ctx = RpcContext.unmarshal(ctx)
        elif data[2] == 'impl_zmq_v2':
            packenv = data[4:]

            msg = unflatten_envelope(packenv)
            request = rpc_common.deserialize_msg(msg)

            # Unmarshal only after verifying the message.
            ctx = RpcContext.unmarshal(data[3])
        else:
            LOG.error(_("ZMQ Envelope version unsupported or unknown."))
            return

        self.pool.spawn_n(self.process, proxy, ctx, request)

    def _process_data(self, ctxt, version, method, args):
        """Process a message in a new thread.

        If the proxy object we have has a dispatch method
        (see rpc.dispatcher.RpcDispatcher), pass it the version,
        method, and args and let it dispatch as appropriate.  If not, use
        the old behavior of magically calling the specified method on the
        proxy we have here.
        """
        ctxt.update_store()
        try:
            rval = self.proxy.dispatch(ctxt, version, method, **args)
            # Check if the result was a generator
            if inspect.isgenerator(rval):
                for x in rval:
                    ctxt.reply(x, None, connection_pool=self.connection_pool)
            else:
                ctxt.reply(rval, None, connection_pool=self.connection_pool)
            # This final None tells multicall that it is done.
            ctxt.reply(ending=True, connection_pool=self.connection_pool)
        except rpc_common.ClientException as e:
            LOG.debug(_('Expected exception during message handling (%s)') %
                      e._exc_info[1])
            ctxt.reply(None, e._exc_info,
                       connection_pool=self.connection_pool,
                       log_failure=False)
        except Exception:
            # sys.exc_info() is deleted by LOG.exception().
            exc_info = sys.exc_info()
            LOG.error(_('Exception during message handling'),
                      exc_info=exc_info)
            ctxt.reply(None, exc_info, connection_pool=self.connection_pool)

    def _check(self, match, target_dict, cred_dict):
        try:
            match_kind, match_value = match.split(':', 1)
        except Exception:
            LOG.exception(_("Failed to understand rule %(match)r") % locals())
            # If the rule is invalid, fail closed
            return False

        func = None
        try:
            old_func = getattr(self, '_check_%s' % match_kind)
        except AttributeError:
            func = self._checks.get(match_kind, self._checks.get(None, None))
        else:
            LOG.warning(_("Inheritance-based rules are deprecated; update "
                          "_check_%s") % match_kind)
            func = lambda brain, kind, value, target, cred: old_func(value,
                                                                     target,
                                                                     cred)

        if not func:
            LOG.error(_("No handler for matches of kind %s") % match_kind)
            # Fail closed
            return False

        return func(self, match_kind, match_value, target_dict, cred_dict)

 def _connect(self, params):
     """Connect to rabbit.  Re-establish any queues that may have
     been declared before if we are reconnecting.  Exceptions should
     be handled by the caller.
     """
     if self.connection:
         LOG.info(_("Reconnecting to AMQP server on "
                  "%(hostname)s:%(port)d") % params)
         try:
             self.connection.release()
         except self.connection_errors:
             pass
         # Setting this in case the next statement fails, though
         # it shouldn't be doing any network operations, yet.
         self.connection = None
     self.connection = kombu.connection.BrokerConnection(**params)
     self.connection_errors = self.connection.connection_errors
     if self.memory_transport:
         # Kludge to speed up tests.
         self.connection.transport.polling_interval = 0.0
     self.consumer_num = itertools.count(1)
     self.connection.connect()
     self.channel = self.connection.channel()
     # work around 'memory' transport bug in 1.1.3
     if self.memory_transport:
         self.channel._new_queue('ae.undeliver')
     for consumer in self.consumers:
         consumer.reconnect(self.channel)
     LOG.info(_('Connected to AMQP server on %(hostname)s:%(port)d') %
              params)

    def update(self, req, id, body):
        LOG.debug(_('Updating app node by id %s' % id))
        context = req.environ['vsm.context']

        if not self.is_valid_body(body, 'appnode'):
            raise exc.HTTPBadRequest()

        body = body.get('appnode')
        LOG.debug('PUT body: %s' % body)

        if id is None:
            expl = _('Request body and URI mismatch: No appnode_id')
            raise webob.exc.HTTPBadRequest(explanation=expl)
        # convert from unicode to str
        id = str(id)
        os_tenant_name = body.get('os_tenant_name', None)
        os_username = body.get('os_username', None)
        os_password = body.get('os_password', None)
        os_auth_url = body.get('os_auth_url', None)

        if not os_tenant_name or not os_username or not os_password or not os_auth_url:
            expl = _('Request body and URI mismatch: os_tenant_name or os_username or os_password or os_auth_url required.')
            raise webob.exc.HTTPBadRequest(explanation=expl)

        appnodes.update(context, id, body)
        return webob.Response(status_int=201)

def _multi_send(method, context, topic, msg, timeout=None,
                envelope=False, _msg_id=None):
    """
    Wraps the sending of messages,
    dispatches to the matchmaker and sends
    message to all relevant hosts.
    """
    conf = CONF
    LOG.debug(_("%(msg)s") % {'msg': ' '.join(map(pformat, (topic, msg)))})

    queues = _get_matchmaker().queues(topic)
    LOG.debug(_("Sending message(s) to: %s"), queues)

    # Don't stack if we have no matchmaker results
    if len(queues) == 0:
        LOG.warn(_("No matchmaker results. Not casting."))
        # While not strictly a timeout, callers know how to handle
        # this exception and a timeout isn't too big a lie.
        raise rpc_common.Timeout(_("No match from matchmaker."))

    # This supports brokerless fanout (addresses > 1)
    for queue in queues:
        (_topic, ip_addr) = queue
        _addr = "tcp://%s:%s" % (ip_addr, conf.rpc_zmq_port)

        if method.__name__ == '_cast':
            eventlet.spawn_n(method, _addr, context,
                             _topic, msg, timeout, envelope,
                             _msg_id)
            return
        return method(_addr, context, _topic, msg, timeout,
                      envelope)

    def __call__(self, message_data):
        """Consumer callback to call a method on a proxy object.

        Parses the message for validity and fires off a thread to call the
        proxy object method.

        Message data should be a dictionary with two keys:
            method: string representing the method to call
            args: dictionary of arg: value

        Example: {'method': 'echo', 'args': {'value': 42}}

        """
        # It is important to clear the context here, because at this point
        # the previous context is stored in local.store.context
        if hasattr(local.store, 'context'):
            del local.store.context
        rpc_common._safe_log(LOG.debug, _('received %s'), message_data)
        self.msg_id_cache.check_duplicate_message(message_data)
        ctxt = unpack_context(self.conf, message_data)
        method = message_data.get('method')
        args = message_data.get('args', {})
        version = message_data.get('version', None)
        if not method:
            LOG.warn(_('no method for message: %s') % message_data)
            ctxt.reply(_('No method for message: %s') % message_data,
                       connection_pool=self.connection_pool)
            return
        self.pool.spawn_n(self._process_data, ctxt, version, method, args)

    def create(self, req, body):
        """create app nodes."""
        LOG.debug(_('Creating new app nodes %s'), body)

        if not self.is_valid_body(body, 'appnodes'):
            raise exc.HTTPBadRequest()

        context = req.environ['vsm.context']
        ip_list = body['appnodes']
        if not isinstance(ip_list, list):
            expl = _('Invalid Request body: should be a list of IP address.')
            raise webob.exc.HTTPBadRequest(explanation=expl)

        node_list = appnodes.create(context, ip_list)
        node_view = self._view_builder.index(node_list)
        for node in node_list:
            #LOG.info('Node %s, Node id: %s, Node ip: %s' % (node, node.id, node.ip))
            #ssh = utils.SSHClient(ip='10.239.131.170', user='root', key_file=r'~/.ssh/id_rsa')
            #ssh = utils.SSHClient(node.ip, 'root', '~/.ssh/id_rsa')
            #ret = ssh.check_ssh(retries=1)
            #status = 'reachable' if ret else 'unreachable'
            status = 'reachable'
            appnodes.update(context, node.id, status)

        return node_view

    def create_consumer(self, topic, proxy, fanout=False):
        # Register with matchmaker.
        _get_matchmaker().register(topic, CONF.rpc_zmq_host)

        # Subscription scenarios
        if fanout:
            sock_type = zmq.SUB
            subscribe = ('', fanout)[type(fanout) == str]
            topic = 'fanout~' + topic.split('.', 1)[0]
        else:
            sock_type = zmq.PULL
            subscribe = None
            topic = '.'.join((topic.split('.', 1)[0], CONF.rpc_zmq_host))

        if topic in self.topics:
            LOG.info(_("Skipping topic registration. Already registered."))
            return

        # Receive messages from (local) proxy
        inaddr = "ipc://%s/zmq_topic_%s" % \
            (CONF.rpc_zmq_ipc_dir, topic)

        LOG.debug(_("Consumer is a zmq.%s"),
                  ['PULL', 'SUB'][sock_type == zmq.SUB])

        self.reactor.register(proxy, inaddr, sock_type,
                              subscribe=subscribe, in_bind=False)
        self.topics.append(topic)

            def publisher(waiter):
                LOG.info(_("Creating proxy for topic: %s"), topic)

                try:
                    # The topic is received over the network,
                    # don't trust this input.
                    if self.badchars.search(topic) is not None:
                        emsg = _("Topic contained dangerous characters.")
                        LOG.warn(emsg)
                        raise RPCException(emsg)

                    out_sock = ZmqSocket("ipc://%s/zmq_topic_%s" %
                                         (ipc_dir, topic),
                                         sock_type, bind=True)
                except RPCException:
                    waiter.send_exception(*sys.exc_info())
                    return

                self.topic_proxy[topic] = eventlet.queue.LightQueue(
                    CONF.rpc_zmq_topic_backlog)
                self.sockets.append(out_sock)

                # It takes some time for a pub socket to open,
                # before we can have any faith in doing a send() to it.
                if sock_type == zmq.PUB:
                    eventlet.sleep(.5)

                waiter.send(True)

                while(True):
                    data = self.topic_proxy[topic].get()
                    out_sock.send(data)
                    LOG.debug(_("ROUTER RELAY-OUT SUCCEEDED %(data)s") %
                              {'data': data})

    def register(self, proxy, in_addr, zmq_type_in, out_addr=None,
                 zmq_type_out=None, in_bind=True, out_bind=True,
                 subscribe=None):

        LOG.info(_("Registering reactor"))

        if zmq_type_in not in (zmq.PULL, zmq.SUB):
            raise RPCException("Bad input socktype")

        # Items push in.
        inq = ZmqSocket(in_addr, zmq_type_in, bind=in_bind,
                        subscribe=subscribe)

        self.proxies[inq] = proxy
        self.sockets.append(inq)

        LOG.info(_("In reactor registered"))

        if not out_addr:
            return

        if zmq_type_out not in (zmq.PUSH, zmq.PUB):
            raise RPCException("Bad output socktype")

        # Items push out.
        outq = ZmqSocket(out_addr, zmq_type_out, bind=out_bind)

        self.mapping[inq] = outq
        self.mapping[outq] = inq
        self.sockets.append(outq)

        LOG.info(_("Out reactor registered"))

    def __init__(self, addr, zmq_type, bind=True, subscribe=None):
        self.sock = _get_ctxt().socket(zmq_type)
        self.addr = addr
        self.type = zmq_type
        self.subscriptions = []

        # Support failures on sending/receiving on wrong socket type.
        self.can_recv = zmq_type in (zmq.PULL, zmq.SUB)
        self.can_send = zmq_type in (zmq.PUSH, zmq.PUB)
        self.can_sub = zmq_type in (zmq.SUB, )

        # Support list, str, & None for subscribe arg (cast to list)
        do_sub = {
            list: subscribe,
            str: [subscribe],
            type(None): []
        }[type(subscribe)]

        for f in do_sub:
            self.subscribe(f)

        str_data = {'addr': addr, 'type': self.socket_s(),
                    'subscribe': subscribe, 'bind': bind}

        LOG.debug(_("Connecting to %(addr)s with %(type)s"), str_data)
        LOG.debug(_("-> Subscribed to %(subscribe)s"), str_data)
        LOG.debug(_("-> bind: %(bind)s"), str_data)

        try:
            if bind:
                self.sock.bind(addr)
            else:
                self.sock.connect(addr)
        except Exception:
            raise RPCException(_("Could not open socket."))

 def _error_callback(exc):
     if isinstance(exc, qpid_exceptions.Empty):
         LOG.debug(_('Timed out waiting for RPC response: %s') %
                   str(exc))
         raise rpc_common.Timeout()
     else:
         LOG.exception(_('Failed to consume message from queue: %s') %
                       str(exc))

def update(contxt, appnode_id, appnode):
    """update app node ssh status, log info or deleted"""
    if contxt is None:
        contxt = context.get_admin_context()

    id = utils.int_from_str(appnode_id)
    LOG.debug("app node id: %s " % id)

    os_tenant_name = appnode["os_tenant_name"]
    os_username = appnode["os_username"]
    os_password = appnode["os_password"]
    os_auth_url = appnode["os_auth_url"]
    os_region_name = appnode["os_region_name"]
    xtrust_user = appnode["xtrust_user"]

    novaclient = nc.Client(os_username, os_password, os_tenant_name, os_auth_url, region_name=os_region_name)
    nova_services = novaclient.services.list()
    nova_compute_hosts = []
    for nova_service in nova_services:
        if nova_service.binary == "nova-compute":
            nova_compute_hosts.append(nova_service.host)

    cinderclient = cc.Client(os_username, os_password, os_tenant_name, os_auth_url, region_name=os_region_name)
    cinder_services = cinderclient.services.list()
    cinder_volume_hosts = []
    for cinder_service in cinder_services:
        if cinder_service.binary == "cinder-volume":
            cinder_volume_hosts.append(cinder_service.host)

    hosts = list(set(nova_compute_hosts + cinder_volume_hosts))
    print hosts

    for host in hosts:
        result, err = utils.execute("check_xtrust_crudini", xtrust_user, host, run_as_root=True)
        LOG.info("==============result: %s" % result)
        LOG.info("==============result: %s" % err)
        if "command not found" in err:
            raise Exception("Command not found on %s" % host)
        if "Permission denied" in err:
            raise Exception("Please check the mutual trust between vsm nodes and openstack nodes")

    """validate openstack access info"""
    try:
        token_url_id = _get_token(os_tenant_name, os_username, os_password, os_auth_url, os_region_name)
        if token_url_id != None:
            appnode["ssh_status"] = "reachable"
        else:
            appnode["ssh_status"] = "no cinder-volume"
    except:
        LOG.exception(_("Error to access to openstack"))
        appnode["ssh_status"] = "unreachable"

    try:
        return db.appnodes_update(contxt, id, appnode)
    except db_exc.DBError as e:
        LOG.exception(_("DB Error on updating Appnodes %s" % e))
        raise exception.AppNodeFailure()

 def _error_callback(exc):
     if isinstance(exc, socket.timeout):
         LOG.debug(_('Timed out waiting for RPC response: %s') %
                   str(exc))
         raise rpc_common.Timeout()
     else:
         LOG.exception(_('Failed to consume message from queue: %s') %
                       str(exc))
         info['do_consume'] = True

 def delete(self, req, id):
     """delete app node by id."""
     LOG.debug(_('Removing app node by id %s' % id))
     context = req.environ['vsm.context']
     if id is None:
         expl = _('Request body and URI mismatch')
         raise webob.exc.HTTPBadRequest(explanation=expl)
     id = str(id)
     appnodes.destroy(context, id)
     return webob.Response(status_int=201)