def _success_start_action(cls, task, status, progress):
     # check if all nodes are ready
     if any(map(lambda n: n.status == 'error',
                task.cluster.nodes)):
         cls._error_start_action(task, 'error', 100)
         return
     task_name = task.name.title()
     task_cache=cls.get_task_cache(task)
     try:      
         message = (
             u"The Role {0} of cluster '{1}' is success {2}"
         ).format(
             task_cache["role"],
             task.cluster.name,
             task_cache[task_cache["role"].lower()]["action"]
         )
     except Exception as exc:
         logger.error(": ".join([
             str(exc),
             traceback.format_exc()
         ]))
         message = u"{0} of environment '{1}' is done".format(
             task_name,
             task.cluster.name
         )
             
     notifier.notify(
         "done",
         message,
         task.cluster_id
     )
     data = {'status': status, 'progress': progress, 'message': message,'timestamp':datetime.datetime.now()}
     objects.Task.update(task, data)

    def _error_action(cls, task, status, progress, message=None):
        task_name = task.name.title()
        if message:
            message = u"{0} has failed. {1}".format(task_name, message)
            # in case we are sending faild task message from astute
            # we should not create a notification with it, because its add
            # a lot of clutter for user
            notify_message = message.split('\n\n')[0]
        else:
            message = u"{0} has failed. Check these nodes:\n{1}".format(
                task_name,
                cls._generate_error_message(
                    task,
                    error_types=('deploy', 'provision'),
                    names_only=True
                )
            )
            notify_message = message

        notifier.notify(
            "error",
            notify_message,
            task.cluster_id
        )
        data = {'status': status, 'progress': progress, 'message': message}
        objects.Task.update(task, data)

    def update_config_resp(cls, **kwargs):
        """Updates task and nodes states at the end of upload config task"""
        logger.info("RPC method update_config_resp received: %s" % jsonutils.dumps(kwargs))

        task_uuid = kwargs["task_uuid"]
        message = kwargs.get("error")
        status = kwargs.get("status")
        progress = kwargs.get("progress")

        task = objects.Task.get_by_uuid(task_uuid, fail_if_not_found=True, lock_for_update=True)

        q_nodes = objects.NodeCollection.filter_by_id_list(None, task.cache["nodes"])
        # lock nodes for updating
        nodes = objects.NodeCollection.lock_for_update(q_nodes).all()

        if status in (consts.TASK_STATUSES.ready, consts.TASK_STATUSES.error):
            for node in nodes:
                node.status = consts.NODE_STATUSES.ready
                node.progress = 100

        if status == consts.TASK_STATUSES.error:
            message = (u"Failed to update configuration on nodes:" u" {0}.").format(
                ", ".join(node.name for node in nodes)
            )
            logger.error(message)
            notifier.notify("error", message)

        db().flush()

        data = {"status": status, "progress": progress, "message": message}
        objects.Task.update(task, data)

        cls._update_action_log_entry(status, task.name, task_uuid, nodes)

    def run(self):
        super(FakeDeletionThread, self).run()
        receiver = NailgunReceiver
        kwargs = {
            'task_uuid': self.task_uuid,
            'nodes': self.data['args']['nodes'],
            'status': 'ready'
        }
        nodes_to_restore = self.data['args'].get('nodes_to_restore', [])
        resp_method = getattr(receiver, self.respond_to)
        resp_method(**kwargs)

        for node_data in nodes_to_restore:
            node = Node(**node_data)

            # Offline node just deleted from db
            # and could not recreated with status
            # discover
            if not node.online:
                continue

            node.status = 'discover'
            db().add(node)
            db().commit()
            node.attributes = NodeAttributes(node_id=node.id)
            node.attributes.volumes = node.volume_manager.gen_volumes_info()
            NetworkManager.update_interfaces_info(node)
            db().commit()

            ram = round(node.meta.get('ram') or 0, 1)
            cores = node.meta.get('cores') or 'unknown'
            notifier.notify("discover",
                            "New node with %s CPU core(s) "
                            "and %s GB memory is discovered" %
                            (cores, ram), node_id=node.id)

    def stop_deployment_resp(cls, **kwargs):
        logger.info("RPC method stop_deployment_resp received: %s" % jsonutils.dumps(kwargs))
        task_uuid = kwargs.get("task_uuid")
        nodes = kwargs.get("nodes", [])
        ia_nodes = kwargs.get("inaccessible_nodes", [])
        message = kwargs.get("error")
        status = kwargs.get("status")
        progress = kwargs.get("progress")

        task = objects.Task.get_by_uuid(task_uuid, fail_if_not_found=True)

        stopping_task_names = [consts.TASK_NAMES.deploy, consts.TASK_NAMES.deployment, consts.TASK_NAMES.provision]

        q_stop_tasks = objects.TaskCollection.filter_by_list(None, "name", stopping_task_names)
        q_stop_tasks = objects.TaskCollection.filter_by(q_stop_tasks, cluster_id=task.cluster_id)
        stop_tasks = objects.TaskCollection.order_by(q_stop_tasks, "id").all()

        # Locking cluster
        objects.Cluster.get_by_uid(task.cluster_id, fail_if_not_found=True, lock_for_update=True)

        if not stop_tasks:
            logger.warning(
                "stop_deployment_resp: deployment tasks \
                            not found for environment '%s'!",
                task.cluster_id,
            )

        if status == consts.TASK_STATUSES.ready:
            task.cluster.status = consts.CLUSTER_STATUSES.stopped

            if stop_tasks:
                map(db().delete, stop_tasks)

            node_uids = [n["uid"] for n in itertools.chain(nodes, ia_nodes)]
            q_nodes = objects.NodeCollection.filter_by_id_list(None, node_uids)
            q_nodes = objects.NodeCollection.filter_by(q_nodes, cluster_id=task.cluster_id)
            q_nodes = objects.NodeCollection.order_by(q_nodes, "id")
            q_nodes = objects.NodeCollection.lock_for_update(q_nodes)

            # locking Nodes for update
            update_nodes = objects.NodeCollection.lock_for_update(q_nodes).all()

            for node in update_nodes:
                objects.Node.reset_to_discover(node)

            if ia_nodes:
                cls._notify_inaccessible(task.cluster_id, [n["uid"] for n in ia_nodes], u"deployment stopping")

            message = (
                u"Deployment of environment '{0}' was successfully stopped. "
                u"Please make changes and reset the environment "
                u"if you want to redeploy it.".format(task.cluster.name or task.cluster_id)
            )

            notifier.notify("done", message, task.cluster_id)

        data = {"status": status, "progress": progress, "message": message}
        objects.Task.update(task, data)

        cls._update_action_log_entry(status, task.name, task_uuid, nodes)

 def _error_start_action(cls, task, status, progress, message=None):
     task_cache=cls.get_task_cache(task)
     task_name = task.name.title()
     if message:
         message = u"The Role {0} of cluster {1} has failed {2}.".format(
             task_cache["role"],
             task.cluster.name,
             task_cache[task_cache["role"].lower()]["action"]
         )
     else:
         message = u"The Role {0} of cluster {1} has failed {2}. Check these nodes:\n{3}".format(
             task_cache["role"],
             task.cluster.name,
             task_cache[task_cache["role"].lower()]["action"],
             cls._generate_error_message(
                 task,
                 error_types=('deploy', 'provision'),
                 names_only=True
             )
         )
     notifier.notify(
         "error",
         message,
         task.cluster_id
     )
     data = {'status': status, 'progress': progress, 'message': message,'timestamp':datetime.datetime.now()}
     objects.Task.update(task, data)
     cls.rollback_role_status(task)

    def remove_cluster_resp(cls, **kwargs):
        logger.info(
            "RPC method remove_cluster_resp received: %s" %
            jsonutils.dumps(kwargs)
        )
        task_uuid = kwargs.get('task_uuid')

        # in remove_nodes_resp method all objects are already locked
        cls.remove_nodes_resp(**kwargs)

        task = objects.Task.get_by_uuid(task_uuid, fail_if_not_found=True)
        cluster = task.cluster

        if task.status in ('ready',):
            logger.debug("Removing environment itself")
            cluster_name = cluster.name

            ips = db().query(IPAddr).filter(
                IPAddr.network.in_([n.id for n in cluster.network_groups])
            )
            for ip in ips:
                db().delete(ip)
            db().flush()

            nm = objects.Cluster.get_network_manager(cluster)
            admin_nets = nm.get_admin_networks()
            objects.Task.delete(task)
            for task_ in cluster.tasks:
                if task_ != task:
                    objects.Transaction.delete(task_)

            objects.Cluster.delete(cluster)
            if admin_nets != nm.get_admin_networks():
                # import it here due to cyclic dependencies problem
                from nailgun.task.manager import UpdateDnsmasqTaskManager
                UpdateDnsmasqTaskManager().execute()

            notifier.notify(
                "done",
                u"Environment '%s' and all its nodes are deleted" % (
                    cluster_name
                )
            )

        elif task.status in ('error',):
            cluster.status = 'error'
            db().add(cluster)
            db().flush()
            if not task.message:
                task.message = "Failed to delete nodes:\n{0}".format(
                    cls._generate_error_message(
                        task,
                        error_types=('deletion',)
                    )
                )
            notifier.notify(
                "error",
                task.message,
                cluster.id
            )

    def PUT(self):
        """:returns: node id.

        :http: * 200 (node are successfully updated)
               * 304 (node data not changed since last request)
               * 400 (data validation failed)
               * 404 (node not found)
        """
        nd = self.checked_data(
            self.validator.validate_update,
            data=web.data())

        node = self.collection.single.get_by_meta(nd)

        if not node:
            raise self.http(404, "Can't find node: {0}".format(nd))

        node.timestamp = datetime.now()

        if not node.online:
            node.online = True
            msg = u"Node '{0}' is back online".format(node.human_readable_name)
            logger.info(msg)
            notifier.notify("discover", msg, node_id=node.id)
        db().flush()

        if 'agent_checksum' in nd and (
            node.agent_checksum == nd['agent_checksum']
        ):
            return {'id': node.id, 'cached': True}

        self.collection.single.update_by_agent(node, nd)
        return {"id": node.id}

 def _download_release_error(cls, release_id, error_message):
     release = db().query(Release).get(release_id)
     release.state = "error"
     db().flush()
     # TODO(NAME): remove this ugly checks
     if error_message != "Task aborted":
         notifier.notify("error", error_message)

    def checked_data(self, validate_method=None, **kwargs):
        try:
            data = kwargs.pop('data', web.data())
            if validate_method:
                method = validate_method
            else:
                method = self.validator.validate

            valid_data = method(data, **kwargs)
        except (
            errors.InvalidInterfacesInfo,
            errors.InvalidMetadata
        ) as exc:
            notifier.notify("error", str(exc))
            raise web.badrequest(message=str(exc))
        except (
            errors.AlreadyExists
        ) as exc:
            err = web.conflict()
            err.message = exc.message
            raise err
        except (
            errors.InvalidData,
            Exception
        ) as exc:
            raise web.badrequest(message=str(exc))
        return valid_data

    def POST(self, cluster_id):
        """:returns: Http response.
        :http: * 201 (nodes are successfully assigned)
               * 400 (invalid nodes data specified)
        """
        data = self.checked_data(
            self.validator.validate_collection_update,
            cluster_id=cluster_id
        )
        nodes = self.get_objects_list_or_404(Node, data.keys())
        cluster = self.get_object_or_404(Cluster, cluster_id)
        for node in nodes:
            node.cluster = cluster
            node.pending_roles = data[node.id]
            node.pending_addition = True
            try:
                node.attributes.volumes = \
                    node.volume_manager.gen_volumes_info()
                node.cluster.add_pending_changes("disks", node_id=node.id)

                network_manager = node.cluster.network_manager
                network_manager.assign_networks_by_default(node)
            except Exception as exc:
                logger.warning(traceback.format_exc())
                notifier.notify(
                    "error",
                    u"Failed to generate attributes for node '{0}': '{1}'"
                    .format(
                        node.human_readable_name(),
                        str(exc) or u"see logs for details"
                    ),
                    node_id=node.id
                )
            db().commit()
        raise web.ok

    def remove_cluster_resp(cls, **kwargs):
        logger.info("RPC method remove_cluster_resp received: %s" % jsonutils.dumps(kwargs))
        task_uuid = kwargs.get("task_uuid")

        # in remove_nodes_resp method all objects are already locked
        cls.remove_nodes_resp(**kwargs)

        task = objects.Task.get_by_uuid(task_uuid, fail_if_not_found=True)
        cluster = task.cluster

        if task.status in ("ready",):
            logger.debug("Removing environment itself")
            cluster_name = cluster.name

            ips = db().query(IPAddr).filter(IPAddr.network.in_([n.id for n in cluster.network_groups]))
            map(db().delete, ips)
            db().flush()

            cls._remove_cluster_relationdata(cluster.id)

            db().delete(cluster)
            db().flush()

            notifier.notify("done", u"环境 '%s' 和所包含的所有节点都已经被删除" % (cluster_name))

        elif task.status in ("error",):
            cluster.status = "error"
            db().add(cluster)
            db().flush()
            if not task.message:
                task.message = "Failed to delete nodes:\n{0}".format(
                    cls._generate_error_message(task, error_types=("deletion",))
                )
            notifier.notify("error", task.message, cluster.id)

    def check_redhat_credentials_resp(cls, **kwargs):
        logger.info("RPC method check_redhat_credentials_resp received: %s" % json.dumps(kwargs))
        task_uuid = kwargs.get("task_uuid")
        error_msg = kwargs.get("error")
        status = kwargs.get("status")
        progress = kwargs.get("progress")

        task = db().query(Task).filter_by(uuid=task_uuid).first()
        if not task:
            logger.error(
                "check_redhat_credentials_resp: task \
                    with UUID %s not found!",
                task_uuid,
            )
            return

        release_info = task.cache["args"]["release_info"]
        release_id = release_info["release_id"]
        release = db().query(Release).get(release_id)
        if not release:
            logger.error("download_release_resp: Release" " with ID %s not found", release_id)
            return

        if error_msg:
            status = "error"
            cls._update_release_state(release_id, "error")
            # TODO(NAME): remove this ugly checks
            if "Unknown error" in error_msg:
                error_msg = "Failed to check Red Hat " "credentials"
            if error_msg != "Task aborted":
                notifier.notify("error", error_msg)

        result = {"release_info": {"release_id": release_id}}

        TaskHelper.update_task_status(task_uuid, status, progress, error_msg, result)

    def dump_environment_resp(cls, **kwargs):
        logger.info(
            "RPC method dump_environment_resp received: %s" %
            jsonutils.dumps(kwargs)
        )
        task_uuid = kwargs.get('task_uuid')
        status = kwargs.get('status')
        progress = kwargs.get('progress')
        error = kwargs.get('error')
        msg = kwargs.get('msg')

        task = objects.Task.get_by_uuid(task_uuid, fail_if_not_found=True)

        if status == 'error':
            notifier.notify('error', error)

            data = {'status': status, 'progress': 100, 'message': error}
            objects.Task.update(task, data)

        elif status == 'ready':
            dumpfile = os.path.basename(msg)
            notifier.notify('done', 'Snapshot is ready. '
                            'Visit Support page to download')
            data = {'status': status, 'progress': progress,
                    'message': '/dump/{0}'.format(dumpfile)}
            objects.Task.update(task, data)

    def fail(self, transaction, reason):
        objects.Transaction.on_finish(
            transaction, consts.TASK_STATUSES.error, message=reason
        )
        helpers.TaskHelper.update_action_log(transaction)
        for sub_transaction in transaction.subtasks:
            if sub_transaction.status == consts.TASK_STATUSES.pending:
                # on_start and on_finish called to properly handle
                # status transition
                objects.Transaction.on_start(sub_transaction)
                objects.Transaction.on_finish(
                    sub_transaction, consts.TASK_STATUSES.error, "Aborted"
                )

        _update_cluster_status(transaction)
        notifier.notify(
            consts.NOTIFICATION_TOPICS.error,
            "Graph execution failed with error: '{0}'."
            "Please check deployment history for more details."
            .format(reason),
            transaction.cluster_id,
            None,
            task_uuid=transaction.uuid
        )
        return True

 def _download_release_completed(cls, release_id):
     release = db().query(Release).get(release_id)
     release.state = 'available'
     db().commit()
     success_msg = u"Successfully downloaded {0}".format(
         release.name
     )
     notifier.notify("done", success_msg)

    def PUT(self, node_id):
        """:returns: JSONized Node object.
        :http: * 200 (OK)
               * 400 (invalid node data specified)
               * 404 (node not found in db)
        """
        node = self.get_object_or_404(Node, node_id)
        if not node.attributes:
            node.attributes = NodeAttributes(node_id=node.id)

        data = self.checked_data(self.validator.validate_update)

        network_manager = NetworkManager()

        old_cluster_id = node.cluster_id

        if data.get("pending_roles") == [] and node.cluster:
            node.cluster.clear_pending_changes(node_id=node.id)

        if "cluster_id" in data:
            if data["cluster_id"] is None and node.cluster:
                node.cluster.clear_pending_changes(node_id=node.id)
                node.roles = node.pending_roles = []
            node.cluster_id = data["cluster_id"]
            if node.cluster_id != old_cluster_id:
                if old_cluster_id:
                    network_manager.clear_assigned_networks(node)
                    network_manager.clear_all_allowed_networks(node.id)
                if node.cluster_id:
                    network_manager.assign_networks_by_default(node)
                    network_manager.allow_network_assignment_to_all_interfaces(node)

        regenerate_volumes = any(
            (
                "roles" in data and set(data["roles"]) != set(node.roles),
                "pending_roles" in data and set(data["pending_roles"]) != set(node.pending_roles),
                node.cluster_id != old_cluster_id,
            )
        )

        for key, value in data.iteritems():
            # we don't allow to update id explicitly
            # and updated cluster_id before all other fields
            if key in ("id", "cluster_id"):
                continue
            setattr(node, key, value)

        if not node.status in ("provisioning", "deploying") and regenerate_volumes:
            try:
                node.attributes.volumes = node.volume_manager.gen_volumes_info()
            except Exception as exc:
                msg = (u"Failed to generate volumes " "info for node '{0}': '{1}'").format(
                    node.name or data.get("mac") or data.get("id"), str(exc) or "see logs for details"
                )
                logger.warning(traceback.format_exc())
                notifier.notify("error", msg, node_id=node.id)
        db().commit()
        return self.render(node)

 def _notify_inaccessible(cls, cluster_id, nodes_uids, action):
     ia_nodes_db = db().query(Node.name).filter(
         Node.id.in_(nodes_uids),
         Node.cluster_id == cluster_id).order_by(Node.id).yield_per(100)
     ia_message = (u"Fuel couldn't reach these nodes during "
                   u"{0}: {1}. Manual check may be needed.".format(
                       action, u", ".join(
                           [u"'{0}'".format(n.name) for n in ia_nodes_db])))
     notifier.notify("warning", ia_message, cluster_id)

 def _error_action(cls, task, status, progress, message=None):
     if message:
         message = u"Deployment has failed. {0}".format(message)
     else:
         message = u"Deployment has failed. Check these nodes:\n{0}".format(
             cls._generate_error_message(task, error_types=("deploy", "provision"), names_only=True)
         )
     notifier.notify("error", message, task.cluster_id)
     TaskHelper.update_task_status(task.uuid, status, progress, message)

    def remove_cluster_resp(cls, **kwargs):
        network_manager = NetworkManager()
        logger.info(
            "RPC method remove_cluster_resp received: %s" %
            json.dumps(kwargs)
        )
        task_uuid = kwargs.get('task_uuid')

        cls.remove_nodes_resp(**kwargs)

        task = db().query(Task).filter_by(uuid=task_uuid).first()
        cluster = task.cluster

        if task.status in ('ready',):
            logger.debug("Removing environment itself")
            cluster_name = cluster.name

            nws = itertools.chain(
                *[n.networks for n in cluster.network_groups]
            )
            ips = db().query(IPAddr).filter(
                IPAddr.network.in_([n.id for n in nws])
            )
            map(db().delete, ips)
            db().commit()

            db().delete(cluster)
            db().commit()

            # Dmitry's hack for clearing VLANs without networks
            network_manager.clear_vlans()

            notifier.notify(
                "done",
                u"Environment '%s' and all its nodes are deleted" % (
                    cluster_name
                )
            )

        elif task.status in ('error',):
            cluster.status = 'error'
            db().add(cluster)
            db().commit()
            if not task.message:
                task.message = "Failed to delete nodes:\n{0}".format(
                    cls._generate_error_message(
                        task,
                        error_types=('deletion',)
                    )
                )
            notifier.notify(
                "error",
                task.message,
                cluster.id
            )