def merge_tops(self, tops):
     '''
     Cleanly merge the top files
     '''
     top = collections.defaultdict(dict)
     orders = collections.defaultdict(dict)
     for ctops in tops.values():
         for ctop in ctops:
             for saltenv, targets in ctop.items():
                 if saltenv == 'include':
                     continue
                 for tgt in targets:
                     matches = []
                     states = OrderedDict()
                     orders[saltenv][tgt] = 0
                     for comp in ctop[saltenv][tgt]:
                         if isinstance(comp, dict):
                             if 'match' in comp:
                                 matches.append(comp)
                             if 'order' in comp:
                                 order = comp['order']
                                 if not isinstance(order, int):
                                     try:
                                         order = int(order)
                                     except ValueError:
                                         order = 0
                                 orders[saltenv][tgt] = order
                         if isinstance(comp, string_types):
                             states[comp] = True
                     top[saltenv][tgt] = matches
                     top[saltenv][tgt].extend(list(states.keys()))
     return self.sort_top_targets(top, orders)

def _prompt_choice(var_name, options):
    '''
    Prompt the user to choose between a list of options, index each one by adding an enumerator
    based on https://github.com/audreyr/cookiecutter/blob/master/cookiecutter/prompt.py#L51

    :param var_name: The question to ask the user
    :type  var_name: ``str``

    :param options: A list of options
    :type  options: ``list`` of ``tupple``

    :rtype: ``tuple``
    :returns: The selected user
    '''
    choice_map = OrderedDict(
        (u'{0}'.format(i), value) for i, value in enumerate(options, 1) if value[0] != 'test'
    )
    choices = choice_map.keys()
    default = u'1'

    choice_lines = [u'{0} - {1} - {2}'.format(c[0], c[1][0], c[1][1]) for c in choice_map.items()]
    prompt = u'\n'.join((
        u'Select {0}:'.format(var_name),
        u'\n'.join(choice_lines),
        u'Choose from {0}'.format(u', '.join(choices))
    ))

    user_choice = click.prompt(
        prompt, type=click.Choice(choices), default=default
    )
    return choice_map[user_choice]

def list_vms(search=None, sort=None, order='uuid,type,ram,state,alias', keyed=True):
    '''
    Return a list of VMs

    search : string
        vmadm filter property
    sort : string
        vmadm sort (-s) property
    order : string
        vmadm order (-o) property -- Default: uuid,type,ram,state,alias
    keyed : boolean
        specified if the output should be an array (False) or dict (True)
            For a dict the key is the first item from the order parameter.
            Note: If key is not unique last vm wins.

    CLI Example:

    .. code-block:: bash

        salt '*' vmadm.list
        salt '*' vmadm.list order=alias,ram,cpu_cap sort=-ram,-cpu_cap
        salt '*' vmadm.list search='type=KVM'
    '''
    ret = {}
    vmadm = _check_vmadm()
    # vmadm list [-p] [-H] [-o field,...] [-s field,...] [field=value ...]
    cmd = '{vmadm} list -p -H {order} {sort} {search}'.format(
        vmadm=vmadm,
        order='-o {0}'.format(order) if order else '',
        sort='-s {0}'.format(sort) if sort else '',
        search=search if search else ''
    )
    res = __salt__['cmd.run_all'](cmd)
    retcode = res['retcode']
    result = OrderedDict() if keyed else []
    if retcode != 0:
        ret['Error'] = res['stderr'] if 'stderr' in res else _exit_status(retcode)
        return ret

    fields = order.split(',')

    for vm in res['stdout'].splitlines():
        vm_data = OrderedDict()
        vm = vm.split(':')
        if keyed:
            for field in fields:
                if fields.index(field) == 0:
                    continue
                vm_data[field.strip()] = vm[fields.index(field)].strip()
            result[vm[0]] = vm_data
        else:
            if len(vm) > 1:
                for field in fields:
                    vm_data[field.strip()] = vm[fields.index(field)].strip()
            else:
                vm_data = vm[0]
            result.append(vm_data)
    return result

def get_vms_per_type(target):
    '''Return all vms indexed by virt_type for a special target'''
    all_targets = OrderedDict()
    for virt_type in VIRT_TYPES:
        per_type = all_targets.setdefault(virt_type, set())
        all_infos = get_targets_and_vms_for_virt_type(virt_type)
        for vmname in all_infos.get(target, []):
            per_type.add(vmname)
    for i in [a for a in all_targets]:
        all_targets[i] = [a for a in all_targets[i]]
    return all_targets

 def gen_ini(self):
     yield '\n[{0}]\n'.format(self.name)
     sections_dict = OrderedDict()
     for name, value in self.iteritems():
         if com_regx.match(name):
             yield '{0}\n'.format(value)
         elif isinstance(value, _Section):
             sections_dict.update({name: value})
         else:
             yield '{0} {1} {2}\n'.format(name, self.sep, value)
     for name, value in sections_dict.iteritems():
         for line in value.gen_ini():
             yield line

    def test_compare_sls_vs_yaml_with_jinja(self):
        tpl = '{{ data }}'
        env = jinja2.Environment()
        src = '{foo: 1, bar: 2, baz: {qux: true}}'

        sls_src = env.from_string(tpl).render(data=yamlex.deserialize(src))
        yml_src = env.from_string(tpl).render(data=yaml.deserialize(src))

        sls_data = yamlex.deserialize(sls_src)
        yml_data = yaml.deserialize(yml_src)

        # ensure that sls & yaml have the same base
        assert isinstance(sls_data, dict)
        assert isinstance(yml_data, dict)
        # The below has been commented out because something the loader test
        # is modifying the yaml renderer to render things to unicode. Without
        # running the loader test, the below passes. Even reloading the module
        # from disk does not reset its internal state (per the Python docs).
        ##
        #assert sls_data == yml_data

        # ensure that sls is ordered, while yaml not
        assert isinstance(sls_data, OrderedDict)
        assert not isinstance(yml_data, OrderedDict)

        # prove that yaml does not handle well with OrderedDict
        # while sls is jinja friendly.
        obj = OrderedDict([
            ('foo', 1),
            ('bar', 2),
            ('baz', {'qux': True})
        ])

        sls_obj = yamlex.deserialize(yamlex.serialize(obj))
        try:
            yml_obj = yaml.deserialize(yaml.serialize(obj))
        except SerializationError:
            # BLAAM! yaml was unable to serialize OrderedDict,
            # but it's not the purpose of the current test.
            yml_obj = obj.copy()

        sls_src = env.from_string(tpl).render(data=sls_obj)
        yml_src = env.from_string(tpl).render(data=yml_obj)

        final_obj = yaml.deserialize(sls_src)
        assert obj == final_obj

        # BLAAM! yml_src is not valid !
        final_obj = OrderedDict(yaml.deserialize(yml_src))
        assert obj != final_obj

 def gen_ini(self):
     yield "{0}[{1}]{0}".format(os.linesep, self.name)
     sections_dict = OrderedDict()
     for name, value in six.iteritems(self):
         if com_regx.match(name):
             yield "{0}{1}".format(value, os.linesep)
         elif isinstance(value, _Section):
             sections_dict.update({name: value})
         else:
             yield "{0}{1}{2}{3}".format(
                 name, (" {0} ".format(self.sep) if self.sep != " " else self.sep), value, os.linesep
             )
     for name, value in six.iteritems(sections_dict):
         for line in value.gen_ini():
             yield line

def get_vms():
    '''Return all vms indexed by targets'''
    data = OrderedDict()
    for virt_type in VIRT_TYPES:
        all_infos = get_targets_and_vms_for_virt_type(virt_type)
        for t in all_infos:
            target = data.setdefault(t, {})
            vms = {}
            vts = set()
            for vmname in all_infos[t]:
                vms.setdefault(vmname, virt_type)
                vts.add(virt_type)
            target['virt_types'] = [a for a in vts]
            target['vms'] = vms
    return data

    def test_compare_sls_vs_yaml_with_jinja(self):
        tpl = '{{ data }}'
        env = jinja2.Environment()
        src = '{foo: 1, bar: 2, baz: {qux: true}}'

        sls_src = env.from_string(tpl).render(data=sls.deserialize(src))
        yml_src = env.from_string(tpl).render(data=yaml.deserialize(src))

        sls_data = sls.deserialize(sls_src)
        yml_data = yaml.deserialize(yml_src)

        # ensure that sls & yaml have the same base
        assert isinstance(sls_data, dict)
        assert isinstance(yml_data, dict)
        assert sls_data == yml_data

        # ensure that sls is ordered, while yaml not
        assert isinstance(sls_data, OrderedDict)
        assert not isinstance(yml_data, OrderedDict)

        # prove that yaml does not handle well with OrderedDict
        # while sls is jinja friendly.
        obj = OrderedDict([
            ('foo', 1),
            ('bar', 2),
            ('baz', {'qux': True})
        ])

        sls_obj = sls.deserialize(sls.serialize(obj))
        try:
            yml_obj = yaml.deserialize(yaml.serialize(obj))
        except SerializationError:
            # BLAAM! yaml was unable to serialize OrderedDict,
            # but it's not the purpose of the current test.
            yml_obj = obj.copy()

        sls_src = env.from_string(tpl).render(data=sls_obj)
        yml_src = env.from_string(tpl).render(data=yml_obj)

        final_obj = yaml.deserialize(sls_src)
        assert obj == final_obj

        # BLAAM! yml_src is not valid !
        final_obj = yaml.deserialize(yml_src)
        assert obj != final_obj

    def element(self, root=None, abspath=None, **kwargs):
        '''
        kwargs contain extra information for custom methods.

        This method must return a valid empty object if no vars are passed
        to allow introspection to create patterns.

        :param root:
        :param abspath:
        '''
        if root is None and abspath is None:
            root = os.path.abspath('.')
            abspath = os.path.abspath('.')
        relpath = os.path.relpath(abspath, root)

        try:
            element = self._elements.get(relpath, OrderedDict())
        except AttributeError:
            element = OrderedDict()

        if not element:
            for field in PATHINFO_FIELDS:
                element.setdefault(field, '')
            element['saltenv'] = kwargs.get('saltenv', 'base')
            element['relpath'] = relpath
            element['abspath'] = abspath
            element['is_pillar'] = kwargs.get('is_pillar', False)

        cachedir = kwargs.get('cachedir', '')
        if cachedir and os.path.commonprefix([abspath, cachedir]) == cachedir:
            element['cache_root'] = root
        else:
            element['file_root'] = root

        element_hook = kwargs.get('_element_hook', None)
        if element_hook:
            element = element_hook(self, element, **kwargs)

        return element

 def merge_tops(self, tops):
     '''
     Cleanly merge the top files
     '''
     top = collections.defaultdict(OrderedDict)
     orders = collections.defaultdict(OrderedDict)
     for ctops in six.itervalues(tops):
         for ctop in ctops:
             for saltenv, targets in six.iteritems(ctop):
                 if saltenv == 'include':
                     continue
                 for tgt in targets:
                     matches = []
                     states = OrderedDict()
                     orders[saltenv][tgt] = 0
                     ignore_missing = False
                     for comp in ctop[saltenv][tgt]:
                         if isinstance(comp, dict):
                             if 'match' in comp:
                                 matches.append(comp)
                             if 'order' in comp:
                                 order = comp['order']
                                 if not isinstance(order, int):
                                     try:
                                         order = int(order)
                                     except ValueError:
                                         order = 0
                                 orders[saltenv][tgt] = order
                             if comp.get('ignore_missing', False):
                                 ignore_missing = True
                         if isinstance(comp, six.string_types):
                             states[comp] = True
                     if ignore_missing:
                         if saltenv not in self.ignored_pillars:
                             self.ignored_pillars[saltenv] = []
                         self.ignored_pillars[saltenv].extend(states.keys())
                     top[saltenv][tgt] = matches
                     top[saltenv][tgt].extend(states)
     return self.sort_top_targets(top, orders)

 def _uncomment_if_commented(self, opt_key):
     # should be called only if opt_key is not already present
     # will uncomment the key if commented and create a place holder
     # for the key where the correct value can be update later
     # used to preserve the ordering of comments and commented options
     # and to make sure options without sectons go above any section
     options_backup = OrderedDict()
     comment_index = None
     for key, value in self.iteritems():
         if comment_index is not None:
             options_backup.update({key: value})
             continue
         if '#comment' not in key:
             continue
         opt_match = opt_regx.match(value.lstrip('#'))
         if opt_match and opt_match.group(2) == opt_key:
             comment_index = key
     for key in options_backup:
         self.pop(key)
     self.pop(comment_index, None)
     super(_Section, self).update({opt_key: None})
     for key, value in options_backup.iteritems():
         super(_Section, self).update({key: value})

def compile(labels, **patterns):  # pylint: disable=W0622
    '''
    Compile patterns.

    Args:
        labels:
    '''
    pattern = patterns.pop('_pattern', None)
    if pattern:
        return pattern

    regex = patterns.pop('_regex', False)
    escape = patterns.pop('_escape', [])

    if not patterns or not labels:
        return None

    for pattern in list(patterns.keys()):
        if pattern not in labels:
            patterns.pop(pattern)

    default_pattern = get_default_pattern(regex)
    escape = escape if escape else []
    _escape_text = functools.partial(escape_text, regex=regex)

    # Set default values and join patterns for each field
    pattern = OrderedDict.fromkeys(labels, None)
    for label in labels:
        if label in patterns and patterns[label]:
            field = patterns[label]
            if isinstance(field, re._pattern_type):  # pylint: disable=W0212
                field = [field.pattern]
            if isinstance(field, six.string_types):
                field = [field]
            if label in escape or not regex:
                field = [_escape_text(text) for text in field]
        else:
            field = default_pattern
        pattern[label] = r'(?:{0})'.format(r'|'.join(field))

    try:
        return re.compile(
            r'\n'.join(six.itervalues(pattern)), re.MULTILINE | re.DOTALL
        )
    except NameError:
        raise

    def __init__(self, match_each=True, **patterns):
        '''
        match_each:
            If True, each file path is matched which prevents uses less memory
            but sacrifices performance a little bit.  If False, the complete
            list is matched after all the file information has been added to
            pathinfo.

        patterns:
            Contains the patterns to match.
            Example:
                { 'saltenv': 'base', 'relpath': ['*.sls'] }
        '''
        super(PathInfoDict, self).__init__(
            fields=PATHINFO_FIELDS,
            match_each=match_each,
            **patterns
        )
        self._elements = OrderedDict()

class StateRegistry(object):
    """
    The StateRegistry holds all of the states that have been created.
    """
    def __init__(self):
        self.empty()

    def empty(self):
        self.states = OrderedDict()
        self.requisites = []

    def salt_data(self):
        states = OrderedDict([
            (id_, state())
            for id_, state in self.states.iteritems()
        ])

        self.empty()

        return states

    def add(self, id_, state):
        if id_ in self.states:
            raise DuplicateState("A state with id '%s' already exists" % id_)

        # if we have requisites in our stack then add them to the state
        if len(self.requisites) > 0:
            for req in self.requisites:
                if req.requisite not in state.kwargs:
                    state.kwargs[req.requisite] = []
                state.kwargs[req.requisite].append(req())

        self.states[id_] = state

    def push_requisite(self, requisite):
        self.requisites.append(requisite)

    def pop_requisite(self):
        del self.requisites[-1]

# -*- coding: utf-8 -*-
'''
Application Kinds of Salt apps.
These are used to indicate what kind of Application is using RAET
'''
from __future__ import absolute_import
from collections import namedtuple
from salt.utils.odict import OrderedDict

# Python equivalent of an enum
APPL_KINDS = OrderedDict([('master', 0),
                          ('minion', 1),
                          ('syndic', 2),
                          ('caller', 3)])
APPL_KIND_NAMES = OrderedDict((v, k) for k, v in list(APPL_KINDS.items()))  # inverse map
ApplKind = namedtuple('ApplKind', list(APPL_KINDS.keys()))
applKinds = ApplKind(**APPL_KINDS)

    def serialize(cls, id_=None):
        # The order matters
        serialized = OrderedDict()
        if id_ is not None:
            # This is meant as a configuration section, sub json schema
            serialized['id'] = '{0}/{1}.json#'.format(BASE_SCHEMA_URL, id_)
        else:
            # Main configuration block, json schema
            serialized['$schema'] = 'http://json-schema.org/draft-04/schema#'
        if cls.title is not None:
            serialized['title'] = cls.title
        if cls.description is not None:
            if cls.description == cls.__doc__:
                serialized['description'] = textwrap.dedent(cls.description).strip()
            else:
                serialized['description'] = cls.description

        required = []
        ordering = []
        serialized['type'] = 'object'
        properties = OrderedDict()
        cls.after_items_update = []
        for name in cls._order:
            skip_order = False
            if name in cls._sections:
                section = cls._sections[name]
                serialized_section = section.serialize(None if section.__flatten__ is True else name)
                if section.__flatten__ is True:
                    # Flatten the configuration section into the parent
                    # configuration
                    properties.update(serialized_section['properties'])
                    if 'x-ordering' in serialized_section:
                        ordering.extend(serialized_section['x-ordering'])
                    if 'required' in serialized_section:
                        required.extend(serialized_section['required'])
                    if hasattr(section, 'after_items_update'):
                        cls.after_items_update.extend(section.after_items_update)
                    skip_order = True
                else:
                    # Store it as a configuration section
                    properties[name] = serialized_section

            if name in cls._items:
                config = cls._items[name]
                # Handle the configuration items defined in the class instance
                if config.__flatten__ is True:
                    serialized_config = config.serialize()
                    cls.after_items_update.append(serialized_config)
                    skip_order = True
                else:
                    properties[name] = config.serialize()

                if config.required:
                    # If it's a required item, add it to the required list
                    required.append(name)

            if skip_order is False:
                # Store the order of the item
                if name not in ordering:
                    ordering.append(name)

        if properties:
            serialized['properties'] = properties

        # Update the serialized object with any items to include after properties
        if cls.after_items_update:
            after_items_update = {}
            for entry in cls.after_items_update:
                name, data = next(six.iteritems(entry))
                if name in after_items_update:
                    after_items_update[name].extend(data)
                else:
                    after_items_update[name] = data
            serialized.update(after_items_update)

        if required:
            # Only include required if not empty
            serialized['required'] = required
        if ordering:
            # Only include ordering if not empty
            serialized['x-ordering'] = ordering
        serialized['additionalProperties'] = cls.__allow_additional_items__
        return serialized

 def empty(self):
     self.states = OrderedDict()
     self.requisites = []

    def handle_func(self, func, data):
        '''
        Execute this method in a multiprocess or thread
        '''
        if salt.utils.is_windows():
            self.functions = salt.loader.minion_mods(self.opts)
            self.returners = salt.loader.returners(self.opts, self.functions)
        ret = {'id': self.opts.get('id', 'master'),
               'fun': func,
               'jid': '{0:%Y%m%d%H%M%S%f}'.format(datetime.datetime.now())}

        proc_fn = os.path.join(
            salt.minion.get_proc_dir(self.opts['cachedir']),
            ret['jid']
        )

        # Check to see if there are other jobs with this
        # signature running.  If there are more than maxrunning
        # jobs present then don't start another.
        # If jid_include is False for this job we can ignore all this
        # NOTE--jid_include defaults to True, thus if it is missing from the data
        # dict we treat it like it was there and is True
        if 'jid_include' not in data or data['jid_include']:
            jobcount = 0
            for basefilename in os.listdir(salt.minion.get_proc_dir(self.opts['cachedir'])):
                fn = os.path.join(salt.minion.get_proc_dir(self.opts['cachedir']), basefilename)
                with salt.utils.fopen(fn, 'r') as fp_:
                    job = salt.payload.Serial(self.opts).load(fp_)
                    log.debug('schedule.handle_func: Checking job against '
                              'fun {0}: {1}'.format(ret['fun'], job))
                    if ret['fun'] == job['fun'] and os_is_running(job['pid']):
                        jobcount += 1
                        log.debug(
                            'schedule.handle_func: Incrementing jobcount, now '
                            '{0}, maxrunning is {1}'.format(
                                      jobcount, data['maxrunning']))
                        if jobcount >= data['maxrunning']:
                            log.debug(
                                'schedule.handle_func: The scheduled job {0} '
                                'was not started, {1} already running'.format(
                                    func, data['maxrunning']))
                            return False

        salt.utils.daemonize_if(self.opts)

        ret['pid'] = os.getpid()

        if 'jid_include' not in data or data['jid_include']:
            log.debug('schedule.handle_func: adding this job to the jobcache '
                      'with data {0}'.format(ret))
            # write this to /var/cache/salt/minion/proc
            with salt.utils.fopen(proc_fn, 'w+') as fp_:
                fp_.write(salt.payload.Serial(self.opts).dumps(ret))

        args = None
        if 'args' in data:
            args = data['args']

        kwargs = None
        if 'kwargs' in data:
            kwargs = data['kwargs']

        try:
            if args and kwargs:
                ret['return'] = self.functions[func](*args, **kwargs)

            if args and not kwargs:
                ret['return'] = self.functions[func](*args)

            if kwargs and not args:
                ret['return'] = self.functions[func](**kwargs)

            if not kwargs and not args:
                ret['return'] = self.functions[func]()

            data_returner = data.get('returner', None)
            if data_returner or self.schedule_returner:
                rets = []
                for returner in [data_returner, self.schedule_returner]:
                    if isinstance(returner, str):
                        rets.append(returner)
                    elif isinstance(returner, list):
                        rets.extend(returner)
                # simple de-duplication with order retained
                rets = OrderedDict.fromkeys(rets).keys()
                for returner in rets:
                    ret_str = '{0}.returner'.format(returner)
                    if ret_str in self.returners:
                        ret['success'] = True
                        self.returners[ret_str](ret)
                    else:
                        log.info(
                            'Job {0} using invalid returner: {1} Ignoring.'.format(
                            func, returner
                            )
                        )
        except Exception:
            log.exception("Unhandled exception running {0}".format(ret['fun']))
            # Although catch-all exception handlers are bad, the exception here
            # is to let the exception bubble up to the top of the thread context,
#.........这里部分代码省略.........

class PathInfoDict(fileinfo.FileInfo):
    def __init__(self, match_each=True, **patterns):
        '''
        match_each:
            If True, each file path is matched which prevents uses less memory
            but sacrifices performance a little bit.  If False, the complete
            list is matched after all the file information has been added to
            pathinfo.

        patterns:
            Contains the patterns to match.
            Example:
                { 'saltenv': 'base', 'relpath': ['*.sls'] }
        '''
        super(PathInfoDict, self).__init__(
            fields=PATHINFO_FIELDS,
            match_each=match_each,
            **patterns
        )
        self._elements = OrderedDict()

    @property
    def as_sequence(self):
        if self.pattern and not self.match_each:
            return list(
                matcher.ifilter(
                    self._elements.values(),
                    _pattern=self.pattern
                )
            )
        return self._elements.values()

    def element(self, root=None, abspath=None, **kwargs):
        '''
        kwargs contain extra information for custom methods.

        This method must return a valid empty object if no vars are passed
        to allow introspection to create patterns.

        :param root:
        :param abspath:
        '''
        if root is None and abspath is None:
            root = os.path.abspath('.')
            abspath = os.path.abspath('.')
        relpath = os.path.relpath(abspath, root)

        try:
            element = self._elements.get(relpath, OrderedDict())
        except AttributeError:
            element = OrderedDict()

        if not element:
            for field in PATHINFO_FIELDS:
                element.setdefault(field, '')
            element['saltenv'] = kwargs.get('saltenv', 'base')
            element['relpath'] = relpath
            element['abspath'] = abspath
            element['is_pillar'] = kwargs.get('is_pillar', False)

        cachedir = kwargs.get('cachedir', '')
        if cachedir and os.path.commonprefix([abspath, cachedir]) == cachedir:
            element['cache_root'] = root
        else:
            element['file_root'] = root

        element_hook = kwargs.get('_element_hook', None)
        if element_hook:
            element = element_hook(self, element, **kwargs)

        return element

    def add_element(self, element, **kwargs):
        add_hook = kwargs.get('_add_hook', None)
        if add_hook:
            element = add_hook(self, element, **kwargs)

        if element['relpath'] not in self._elements:
            self._elements[element['relpath']] = element

    def filelist(self, roots, **kwargs):
        '''
        :param roots:
            file_roots, pillar_roots, cache_roots, etc to walk.

        kwargs:
            Contains any extra variables to pass to element.

        '''
        for env, destdirs in six.iteritems(roots):
            kwargs['saltenv'] = env
            super(PathInfoDict, self).filelist(destdirs, **kwargs)

        return self.as_sequence