def runop(self, op, nodes=None, local_only=False):
     '''
     Execute an operation.
     '''
     if not nodes or self.run_on_all(op):
         nodes = self.nodes
     self.last_op = op
     self.set_rscenv(op)
     real_op = (op == "probe" and "monitor" or op)
     cmd = self.exec_cmd(real_op)
     common_debug("running %s on %s" % (real_op, nodes))
     for attr in self.rscenv.keys():
         # shell doesn't allow "-" in var names
         envvar = attr.replace("-", "_")
         cmd = "%s=%s %s" % (envvar, quote(self.rscenv[attr]), cmd)
     if local_only:
         self.ec_l[this_node()] = ext_cmd(cmd)
     else:
         from crm_pssh import do_pssh_cmd
         statuses = do_pssh_cmd(cmd, nodes, self.outdir, self.errdir, self.timeout)
         for i in range(len(nodes)):
             try:
                 self.ec_l[nodes[i]] = statuses[i]
             except:
                 self.ec_l[nodes[i]] = self.undef
     return

def set_deep_meta_attr_node(target_node, attr, value):
    nvpair_l = []
    if xmlutil.is_clone(target_node):
        for c in target_node.iterchildren():
            if xmlutil.is_child_rsc(c):
                rm_meta_attribute(c, attr, nvpair_l)
    if config.core.manage_children != "never" and \
            (xmlutil.is_group(target_node) or
             (xmlutil.is_clone(target_node) and xmlutil.cloned_el(target_node) == "group")):
        odd_children = get_children_with_different_attr(target_node, attr, value)
        for c in odd_children:
            if config.core.manage_children == "always" or \
                    (config.core.manage_children == "ask" and
                     utils.ask("Do you want to override %s for child resource %s?" %
                               (attr, c.get("id")))):
                common_debug("force remove meta attr %s from %s" %
                             (attr, c.get("id")))
                rm_meta_attribute(c, attr, nvpair_l, force_children=True)
    xmlutil.rmnodes(list(set(nvpair_l)))
    xmlutil.xml_processnodes(target_node,
                             xmlutil.is_emptynvpairs, xmlutil.rmnodes)

    # work around issue with pcs interoperability
    # by finding exising nvpairs -- if there are any, just
    # set the value in those. Otherwise fall back to adding
    # to all meta_attributes tags
    nvpairs = target_node.xpath("./meta_attributes/nvpair[@name='%s']" % (attr))
    if len(nvpairs) > 0:
        for nvpair in nvpairs:
            nvpair.set("value", value)
    else:
        for n in xmlutil.get_set_nodes(target_node, "meta_attributes", 1):
            xmlutil.set_attr(n, attr, value)
    return True

def next_peinputs(node_pe_l, outdir, errdir):
    '''
    pssh to nodes to collect new logs.
    '''
    l = []
    for node, pe_l in node_pe_l:
        r = re.search("(.*)/pengine/", pe_l[0])
        if not r:
            common_err("strange, %s doesn't contain string pengine" % pe_l[0])
            continue
        dir = "/%s" % r.group(1)
        red_pe_l = [x.replace("%s/" % r.group(1), "") for x in pe_l]
        common_debug("getting new PE inputs %s from %s" % (red_pe_l, node))
        cmdline = "tar -C %s -cf - %s" % (dir, ' '.join(red_pe_l))
        myopts = ["-q", "-o", outdir, "-e", errdir]
        opts, args = parse_args(myopts)
        l.append([node, cmdline])
    if not l:
        # is this a failure?
        return True
    statuses = do_pssh(l, opts)
    if statuses:
        return examine_outcome(l, opts, statuses)
    else:
        return False

 def remove(self, node_id):
     if not node_id:
         return
     try:
         del self._id_store[node_id]
         common_debug("id_store: removed %s" % node_id)
     except KeyError:
         pass

def set_deep_meta_attr(rsc, attr, value, commit=True):
    """
    If the referenced rsc is a primitive that belongs to a group,
    then set its attribute.
    Otherwise, go up to the topmost resource which contains this
    resource and set the attribute there (i.e. if the resource is
    cloned).
    If it's a group then check its children. If any of them has
    the attribute set to a value different from the one given,
    then ask the user whether to reset them or not (exact
    behaviour depends on the value of config.core.manage_children).
    """

    def update_obj(obj):
        """
        set the meta attribute in the given object
        """
        node = obj.node
        obj.set_updated()
        if not (node.tag == "primitive" and
                node.getparent().tag == "group"):
            node = xmlutil.get_topmost_rsc(node)
        return set_deep_meta_attr_node(node, attr, value)

    def flatten(objs):
        for obj in objs:
            if isinstance(obj, list):
                for subobj in obj:
                    yield subobj
            else:
                yield obj

    def resolve(obj):
        if obj.obj_type == 'tag':
            return [cib_factory.find_object(o) for o in obj.node.xpath('./obj_ref/@id')]
        return obj

    objs = cib_factory.find_objects(rsc)
    while any(obj for obj in objs if obj.obj_type == 'tag'):
        objs = list(flatten(resolve(obj) for obj in objs))
    common_debug("set_deep_meta_attr: %s" % (', '.join([obj.obj_id for obj in objs])))
    if not objs:
        common_error("Resource not found: %s" % (rsc))
        return False

    ok = all(update_obj(obj) for obj in objs)
    if not ok:
        common_error("Failed to update meta attributes for %s" % (rsc))
        return False

    if not commit:
        return True

    ok = cib_factory.commit()
    if not ok:
        common_error("Failed to commit updates to %s" % (rsc))
        return False
    return True

 def _stonith_types(self):
     rc, l = stdout2list("stonith -L")
     if rc != 0:
         # stonith(8) may not be installed
         common_debug("stonith exited with code %d" % rc)
         l = []
     for ra in os_types_list("/usr/sbin/fence_*"):
         if ra not in ("fence_ack_manual", "fence_pcmk", "fence_legacy"):
             l.append(ra)

def prog_meta(prog):
    '''
    Do external program metadata.
    '''
    if is_program(prog):
        rc, l = stdout2list("%s metadata" % prog)
        if rc == 0:
            return l
        common_debug("%s metadata exited with code %d" % (prog, rc))
    return []

def is_min_pcmk_ver(min_ver, cib_f=None):
    if not constants.pcmk_version:
        if cib_f:
            constants.pcmk_version = get_cib_property(cib_f, "dc-version", "1.1.11")
            common_debug("found pacemaker version: %s in cib: %s" %
                         (constants.pcmk_version, cib_f))
        else:
            constants.pcmk_version = get_pcmk_version("1.1.11")
    from distutils.version import LooseVersion
    return LooseVersion(constants.pcmk_version) >= LooseVersion(min_ver)

 def crm_resource(self, opts):
     '''
     Get information from crm_resource.
     '''
     rc, l = stdout2list("crm_resource %s" % opts, stderr_on=False)
     # not clear when/why crm_resource exits with non-zero
     # code
     if rc != 0:
         common_debug("crm_resource %s exited with code %d" %
                      (opts, rc))
     return l

def get_topnode(cib_elem, tag):
    "Get configuration element or create/append if there's none."
    conf_elem = cib_elem.find("configuration")
    if conf_elem is None:
        common_err("no configuration element found!")
        return None
    if tag == "configuration":
        return conf_elem
    e = cib_elem.find("configuration/%s" % tag)
    if e is None:
        common_debug("create configuration section %s" % tag)
        e = etree.SubElement(conf_elem, tag)
    return e

def pipe_string(cmd, s):
    rc = -1  # command failed
    cmd = add_sudo(cmd)
    common_debug("piping string to %s" % cmd)
    if options.regression_tests:
        print ".EXT", cmd
    p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE)
    try:
        p.communicate(s)
        p.wait()
        rc = p.returncode
    except IOError, msg:
        if "Broken pipe" not in msg:
            common_err(msg)

def running_on(resource):
    "returns list of node names where the given resource is running"
    rsc_locate = "crm_resource --resource '%s' --locate"
    rc, out, err = get_stdout_stderr(rsc_locate % (resource))
    if rc != 0:
        return []
    nodes = []
    head = "resource %s is running on: " % (resource)
    for line in out.split('\n'):
        if line.strip().startswith(head):
            w = line[len(head):].split()
            if w:
                nodes.append(w[0])
    common_debug("%s running on: %s" % (resource, nodes))
    return nodes

def prog_meta(prog):
    '''
    Do external program metadata.
    '''
    if prog == "crmd" and os.path.isfile(os.path.join(config.path.crm_daemon_dir, 'crmd')):
        prog = os.path.join(config.path.crm_daemon_dir, 'crmd')
        rc, l = stdout2list("%s metadata" % prog)
        if rc == 0:
            return l
        common_debug("%s metadata exited with code %d" % (prog, rc))
    elif is_program(prog):
        rc, l = stdout2list("%s metadata" % prog)
        if rc == 0:
            return l
        common_debug("%s metadata exited with code %d" % (prog, rc))
    return []

 def _set_source(self, src, live_from_time=None):
     '''
     Have the last history source survive the History
     and Report instances
     '''
     common_debug("setting source to %s" % src)
     if not self._check_source(src):
         return False
     crm_report.set_source(src)
     options.history = src
     self.current_session = None
     to_time = ''
     if src == "live":
         from_time = time.ctime(live_from_time and live_from_time or (time.time() - 60*60))
     else:
         from_time = ''
     return self._set_period(from_time, to_time)

 def _xdg_file(name, xdg_name, chk_fun, directory):
     from msg import common_warn, common_info, common_debug
     if not name:
         return name
     if not os.path.isdir(directory):
         os.makedirs(directory, 0700)
     new = os.path.join(directory, xdg_name)
     if directory == CONFIG_HOME and chk_fun(new) and chk_fun(name):
         common_warn("both %s and %s exist, please cleanup" % (name, new))
         return name
     if chk_fun(name):
         if directory == CONFIG_HOME:
             common_info("moving %s to %s" % (name, new))
         else:
             common_debug("moving %s to %s" % (name, new))
         os.rename(name, new)
     return new

def next_loglines(a, outdir, errdir):
    '''
    pssh to nodes to collect new logs.
    '''
    l = []
    for node, rptlog, logfile, nextpos in a:
        common_debug("updating %s from %s (pos %d)" %
                     (logfile, node, nextpos))
        cmdline = "perl -e 'exit(%d) if (stat(\"%s\"))[7]<%d' && tail -c +%d %s" % (
            _EC_LOGROT, logfile, nextpos-1, nextpos, logfile)
        opts = parse_args(outdir, errdir)
        l.append([node, cmdline])
    statuses = do_pssh(l, opts)
    if statuses:
        return examine_outcome(l, opts, statuses)
    else:
        return False

def run_ptest(graph_s, nograph, scores, utilization, actions, verbosity):
    '''
    Pipe graph_s thru ptest(8). Show graph using dotty if requested.
    '''
    actions_filter = "grep LogActions: | grep -vw Leave"
    ptest = "2>&1 %s -x -" % config.core.ptest
    if re.search("simulate", ptest) and \
            not re.search("-[RS]", ptest):
        ptest = "%s -S" % ptest
    if verbosity:
        if actions:
            verbosity = 'v' * max(3, len(verbosity))
        ptest = "%s -%s" % (ptest, verbosity.upper())
    if scores:
        ptest = "%s -s" % ptest
    if utilization:
        ptest = "%s -U" % ptest
    if config.core.dotty and not nograph:
        fd, dotfile = mkstemp()
        ptest = "%s -D %s" % (ptest, dotfile)
    else:
        dotfile = None
    # ptest prints to stderr
    if actions:
        ptest = "%s | %s" % (ptest, actions_filter)
    if options.regression_tests:
        ptest = ">/dev/null %s" % ptest
    common_debug("invoke: %s" % ptest)
    rc, s = get_stdout(ptest, input_s=graph_s)
    if rc != 0:
        common_debug("%s exited with %d" % (ptest, rc))
        if actions and rc == 1:
            common_warn("No actions found.")
        else:
            common_warn("Simulation was unsuccessful (RC=%d)." % (rc))
    if dotfile:
        if os.path.getsize(dotfile) > 0:
            show_dot_graph(dotfile)
        else:
            common_warn("ptest produced empty dot file")
    else:
        if not nograph:
            common_info("install graphviz to see a transition graph")
    if s:
        page_string(s)
    return True

def get_pcmk_version(dflt):
    version = dflt

    crmd = is_program('crmd')
    if crmd:
        cmd = crmd
    else:
        return version

    try:
        rc, s = get_stdout("%s version" % (cmd))
        if rc != 0:
            common_err("%s exited with %d" % (cmd, rc))
        else:
            version = s.split()[2]
            common_debug("found pacemaker version: %s" % version)
    except Exception, msg:
        common_warn("could not get the pacemaker version, bad installation?")
        common_warn(msg)

def get_pcmk_version(dflt):
    version = dflt

    if is_program('crmd'):
        cmd = 'crmd'
    elif os.path.isfile(os.path.join(config.path.crm_daemon_dir, 'crmd')):
        cmd = os.path.join(config.path.crm_daemon_dir, 'crmd')
    else:
        return version

    try:
        rc, s = get_stdout("%s version" % (cmd))
        if rc != 0:
            common_err("%s exited with %d" % (cmd, rc))
        else:
            version = s.split()[2]
            common_debug("found pacemaker version: %s" % version)
    except Exception, msg:
        common_warn("could not get the pacemaker version, bad installation?")
        common_warn(msg)

def set_deep_meta_attr_node(target_node, attr, value):
    nvpair_l = []
    if xmlutil.is_clone(target_node):
        for c in target_node.iterchildren():
            if xmlutil.is_child_rsc(c):
                rm_meta_attribute(c, attr, nvpair_l)
    if config.core.manage_children != "never" and \
            (xmlutil.is_group(target_node) or
             (xmlutil.is_clone(target_node) and xmlutil.cloned_el(target_node) == "group")):
        odd_children = get_children_with_different_attr(target_node, attr, value)
        for c in odd_children:
            if config.core.manage_children == "always" or \
                    (config.core.manage_children == "ask" and
                     utils.ask("Do you want to override %s for child resource %s?" %
                               (attr, c.get("id")))):
                common_debug("force remove meta attr %s from %s" %
                             (attr, c.get("id")))
                rm_meta_attribute(c, attr, nvpair_l, force_children=True)
    xmlutil.rmnodes(list(set(nvpair_l)))
    for n in xmlutil.get_set_nodes(target_node, "meta_attributes", 1):
        xmlutil.set_attr(n, attr, value)
    return xmlutil.commit_rsc(target_node)