def test_template_within_template():
    config_str = """
    {
        templates: {
            a_layer: {
                weight_init: $weight_init,
            },
            glorotuniform: 'funny glorot uniform template $range',
        },
        variants: [[
            {
                weight_init: [$glorotuniform], 
                range: [2], 
                layer:[$a_layer],
            },
        ]]
    }
    """
    
    yaml.add_constructor(u'!TransformValsToString', transform_vals_to_string_constructor)
    
    config_obj = yaml.load(config_str.replace("templates:", 
        "templates: !TransformValsToString"))
    
    all_params =  create_variants_recursively(config_obj['variants'])
    templates = config_obj['templates']
    final_params, templates_to_parameters = merge_parameters_and_templates(all_params, templates)
    assert final_params == [{'weight_init': "'funny glorot uniform template 2'\n",
        'range': 2,
        'layer': "{weight_init: 'funny glorot uniform template 2'\n}\n"}]
     
    assert templates_to_parameters ==  [dict(a_layer=set(['weight_init']),
                                   glorotuniform=set(['range']))]

def get_yaml(path):
    """
    Read the file identified by `path` and import its YAML contents.

    :arg path: The path to a YAML configuration file.
    :rtype: dict
    """
    # Set the stage here to parse single scalar value environment vars from
    # the YAML file being read
    single = re.compile( r'^\$\{(.*)\}$' )
    yaml.add_implicit_resolver ( "!single", single )
    def single_constructor(loader,node):
        value = loader.construct_scalar(node)
        proto = single.match(value).group(1)
        default = None
        if len(proto.split(':')) > 1:
            envvar, default = proto.split(':')
        else:
            envvar = proto
        return os.environ[envvar] if envvar in os.environ else default
    yaml.add_constructor('!single', single_constructor)

    raw = read_file(path)
    try:
        cfg = yaml.load(raw)
    except yaml.scanner.ScannerError as e:
        raise ConfigurationError(
            'Unable to parse YAML file. Error: {0}'.format(e))
    return cfg

def parse(u):
    """
    Parse the contents of a spec test file, and return a dict.

    Arguments:

      u: a unicode string.

    """
    # TODO: find a cleaner mechanism for choosing between the two.
    if yaml is None:
        # Then use json.

        # The only way to get the simplejson module to return unicode strings
        # is to pass it unicode.  See, for example--
        #
        #   http://code.google.com/p/simplejson/issues/detail?id=40
        #
        # and the documentation of simplejson.loads():
        #
        #   "If s is a str then decoded JSON strings that contain only ASCII
        #    characters may be parsed as str for performance and memory reasons.
        #    If your code expects only unicode the appropriate solution is
        #    decode s to unicode prior to calling loads."
        #
        return json.loads(u)
    # Otherwise, yaml.

    def code_constructor(loader, node):
        value = loader.construct_mapping(node)
        return eval(value['python'], {})

    yaml.add_constructor('!code', code_constructor)
    return yaml.load(u)

def load(yaml_data, **kwargs):
    ''' convert yaml data into a configuration
         @param yaml_data: yaml data to be parsed
         @param configData: do substitution from this data

         parsing includes two custom yaml tags

         !format does string.format substitution using mapping data in
         the node. kwargs override default values if present
    '''

    def _python_format(loader, node):
        ''' do python string formatting

            requires a mapping input
            special key "format" is the string to be formatted
            all other keys are passed as keyword arguments
        '''

        params = Bunch(loader.construct_mapping(node))

        # allow kwargs substitution
        for key, value in kwargs.items():
            if key in params:
                params[key] = value

        rv = params.format.format(**params) #pylint: disable=W0142
        return rv

    yaml.add_constructor('!format', _python_format)

    return _bunchify_tree(yaml.load(yaml_data))

def do_deploy(repo, deploy_file, target = None):
    repo = path.path(repo)
    if target is None:
        target = DigsbyDeployTarget(repo)

    def path_constructor(loader, node):
        if node.id == 'sequence':
            return repo.joinpath(*(loader.construct_sequence(node))).abspath()
        elif node.id == 'scalar':
            return (repo / loader.construct_scalar(node)).abspath()

    import digsby_phases
    yaml.add_constructor('!path', path_constructor)
    phases = yaml.load(open(deploy_file))

    for phase in phases:
        assert(len(phase) == 1)
        phase_name, phase_parts = phase.items()[0]
        for strat in phase_parts:
            ((strategy_name, options),) = strat.items()
            options = target.get_options(phase_name, strategy_name, options)
            with deploy.phase(phase_name, strategy_name, target, **options) as phase:
                phase.do()

    print('*** done ***')

    def __init__(self, environment_name, nova_descriptor_file=None):
        self._nova_descriptor_file = nova_descriptor_file or 'nova.yml'
        self._environment_name = environment_name
        self._environment = None
        self._codedeploy_app = None

        self.templates_used = dict()
        yaml.add_constructor("!include", yaml_include)

        with open(os.path.join(spec.__path__[0], 'nova_service_schema.yml'), 'r') as schemaYaml:
            schema = yaml.load(schemaYaml)

        v = Validator(schema)
        try:
            with open(self._nova_descriptor_file, 'r') as novaYaml:
                self.service_spec = yaml.safe_load(novaYaml)

            # Validate loaded dictionary
            valid = v.validate(self.service_spec)
            if not valid:
                raise NovaError("Invalid nova service descriptor file '%s': %s" % (self._nova_descriptor_file, v.errors))
            else:
                self.service = Service.load(self.service_spec)
                self.service_name = self.service.name
                self.service_port = self.service.port
                self.service_healthcheck_url = self.service.healthcheck_url
        except IOError:
            raise NovaError("No nova service descriptor found at '%s'" % self._nova_descriptor_file)

    def _load_config(self):
        """
        Load specified config file
        """
        try:
            with open(self.conf['config'], 'r') as fp:
                config = fp.read()
        except IOError as e:
            lg.error("Can't read config file %s: %s" % (self.conf['config'], e))
            raise

        # Register include constructors
        yaml.add_constructor('!include_dir', self._yaml_include_dir)
        yaml.add_constructor('!include', self._yaml_include)

        try:
            conf = yaml.load(config)
        except Exception as e:
            lg.error("Can't parse config file %s: %s" % (self.conf['config'], e))
            raise
        finally:
            fp.close()

        # Store parameters but don't overwite
        # those submitted by command line
        for key, value in conf.iteritems():
            if self.conf.has_key(key):
                # User has submitted own parameter,
                # use that instead of config one
                lg.debug("Using parameter %s from user, ignoring config file value" % key)
            else:
                self.conf[key] = value

def main(argv):
    """Transcribe YAML content into HTML through Django templates."""

    conf = generate_config(argv)

    settings.configure(
        TEMPLATE_DIRS=(conf["templates"],),
        TEMPLATE_LOADERS=(("django.template.loaders.cached.Loader", ("django.template.loaders.filesystem.Loader",)),),
    )
    import django.contrib.syndication.views  # Requires Django to be configured

    django.setup()

    yaml.add_constructor("tag:yaml.org,2002:timestamp", _timestamp_loader)

    recreate_dir(conf["output"])
    copy_static_content(conf["static"], conf["output"])

    for root, _, files in os.walk(conf["content"]):
        all_items = []

        item_root = os.path.relpath(root, conf["content"])
        output_root = os.path.join(conf["output"], item_root)
        if item_root != ".":
            os.mkdir(output_root)

        for file_name in files:
            content = yaml.load(open(os.path.join(root, file_name)))
            content["slug"] = os.path.splitext(file_name)[0]
            all_items.append(content)

        if all_items:
            output_all(all_items, item_root, output_root, conf["meta"])

def load_config(path):

    """validates, loads and configures the yaml document at the specified path

    :param path: the path to the file
    :return: the parsed yaml document
    :raises SchemaError: if the yaml document does not validate
    """

    validator = Core(source_file=path, schema_data=config_schema)
    validator.validate(raise_exception=True)

    pattern = re.compile(r'^(.*)<%= ENV\[\'(.*)\'\] %>(.*)$')
    yaml.add_implicit_resolver('!env_regex', pattern)

    def env_regex(loader, node):
        value = loader.construct_scalar(node)
        front, variable_name, back = pattern.match(value).groups()
        return str(front) + os.environ[variable_name] + str(back)

    yaml.add_constructor('!env_regex', env_regex)

    with open(path, 'r') as stream:
        doc = yaml.load(stream)
        return doc

def __init_module__():
    yaml.add_representer(uuid.UUID, uuid_representer, Dumper=ConfigDumper)
    yaml.add_representer(ConfigPath, path_representer, Dumper=ConfigDumper)
    yaml.add_constructor('!uuid', uuid_constructor, Loader=ConfigLoader)
    yaml.add_constructor(
        '!create-if-needed', path_constructor, Loader=ConfigLoader)
    uuid_add_implicit_resolver()

    def _exec_module(self, module):
        fullname = module.__name__

        if yaml is None:
            raise ImportError('PyYaml is not installed')

        class MyYamlLoader(YamlLoader):
            pass

        for tag, func in iteritems(self._defaults):
            @functools.wraps(func)
            def constructor(loader, node):
                return func(fullname, loader.construct_mapping(node))

            yaml.add_constructor(tag, constructor, Loader=MyYamlLoader)

        try:
            stream = file(self.get_filename(fullname), 'r')
            docs = yaml.load_all(stream, Loader=MyYamlLoader)

        except IOError:
            raise ImportError("IO error while reading a stream")
        except yaml.YAMLError as e:
            raise e  # XXX convert into SyntaxError

        else:
            for doc in docs:
                if not hasattr(doc, '__name__'):
                    continue
                setattr(module, doc.__name__, doc)

 def _load_css_params(self, skin, skindir):
     yaml_path = os.path.join(skindir, "variables.yaml")
     if os.path.isfile(yaml_path):
         with open(yaml_path, 'r') as yamlfd:
             css_text = yamlfd.read()
         try:
             yaml.add_constructor('!secret', ha._secret_yaml, Loader=yaml.SafeLoader)
             css = yaml.load(css_text, Loader=yaml.SafeLoader)
         except yaml.YAMLError as exc:
             ha.log(self.logger, "WARNING", "Error loading CSS variables")
             if hasattr(exc, 'problem_mark'):
                 if exc.context is not None:
                     ha.log(self.logger, "WARNING", "parser says")
                     ha.log(self.logger, "WARNING", str(exc.problem_mark))
                     ha.log(self.logger, "WARNING", str(exc.problem) + " " + str(exc.context))
                 else:
                     ha.log(self.logger, "WARNING", "parser says")
                     ha.log(self.logger, "WARNING", str(exc.problem_mark))
                     ha.log(self.logger, "WARNING", str(exc.problem))
             return None
         if css is None:
             return {}
         else:
             return self._resolve_css_params(css, css)
     else:
         ha.log(self.logger, "WARNING", "Error loading variables.yaml for skin '{}'".format(skin))
         return None

def main():
    requestsexceptions.squelch_warnings()
    yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
                         construct_yaml_map)

    yaml.add_representer(OrderedDict, project_representer,
                         Dumper=IndentedDumper)

    data = yaml.load(open('openstack_catalog/web/static/assets.yaml'))

    assets = []
    for a in data['assets']:
        url = a.get('attributes', {}).get('url')
        if not a.get('active', True) or not url:
            assets.append(a)
            continue

        r = requests.head(url, allow_redirects=True)
        if r.status_code != 200:
            a['active'] = False
        else:
            hash_url = a.get('hash_url')
            if hash_url:
                hashes = get_hashes(hash_url)
                filename = url.split("/")[-1]
                a['attributes']['hash'] = hashes.get(filename, 'unknown')

        assets.append(a)

    output = {'assets': assets}
    with open('openstack_catalog/web/static/assets.yaml', 'w') as out:
        out.write(yaml.dump(output, default_flow_style=False,
                            Dumper=IndentedDumper, width=80,
                            indent=2))

    def _enhanced_yaml_module(self, sysconfig):

        # NOTE: This is soo ugly, sorry for that, in future we need to modify
        # PyYAML to let us specify callbacks, somehow.  But for now, import
        # yaml right here (local import) to be able to add the
        # constructors/representers **only** locally (don't modify global
        # context).
        def _eval_node(loader, node):
            return str(eval(str(loader.construct_scalar(node)), {
                'project': self.project,
                'config': sysconfig,
                'macros': sysconfig['macros'],
            }))

        import yaml
        try:
            yaml.add_constructor(u'!eval', _eval_node, yaml.FullLoader)
            yaml.dg_load = functools.partial(yaml.load, Loader=yaml.FullLoader)
        except AttributeError:
            # Older versions of PyYAML don't have yaml.FullLoader, remove this
            # once we don't have to deal with those.
            yaml.add_constructor(u'!eval', _eval_node)
            yaml.dg_load = yaml.load

        return yaml

def LoadConfigDict(config_paths, model_params):
  """Loads config dictionary from specified yaml files or command line yaml."""

  # Ensure that no duplicate keys can be loaded (causing pain).
  yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
                       NoDuplicatesConstructor)

  # Handle either ',' or '#' separated config lists, since borg will only
  # accept '#'.
  sep = ',' if ',' in config_paths else '#'

  # Load flags from config file.
  final_config = {}
  if config_paths:
    for config_path in config_paths.split(sep):
      config_path = config_path.strip()
      if not config_path:
        continue
      config_path = os.path.abspath(config_path)
      tf.logging.info('Loading config from %s', config_path)
      with tf.gfile.GFile(config_path.strip()) as config_file:
        config_flags = yaml.load(config_file)
        final_config = DeepMergeDict(final_config, config_flags)
  if model_params:
    model_params = MaybeLoadYaml(model_params)
    final_config = DeepMergeDict(final_config, model_params)
  tf.logging.info('Final Config:\n%s', yaml.dump(final_config))
  return final_config

def get_all_host_facter_message():
    # 由于puppet的yaml文件是ruby格式的，需要进行转换
    yaml.add_multi_constructor(u"!ruby/object:", construct_ruby_object)
    yaml.add_constructor(u"!ruby/sym", construct_ruby_sym)
    # 获取所有有facters信息的主机文件名称
    for dirpath, dirnames, filenames in os.walk(yaml_dir):
        # 只需要处理yaml目录下得yaml结尾的文件
        if dirpath == yaml_dir:
            for file in filenames:
                file_name, file_ext = os.path.splitext(file)
                if file_ext == '.yaml':
                    host_yaml_path = yaml_dir + '/' + file
                    # 得到yaml文件内容, 字典形式
                    host_yaml_result_dict = yaml_file_handle(host_yaml_path)
                    # 对单个agent的数据进行处理
                    if host_yaml_result_dict:
                        # 有key为facts,所以可以直接查找facts key值
                        if host_yaml_result_dict.has_key('facts'):
                            data_dict = host_yaml_result_dict['facts']['values']
                        # 没有的直接取
                        else:
                            data_dict = host_yaml_result_dict['values']

                    # 现在就可以data数据进行处理，取得我们所需要得数据
                    result_dict = handle_facter_message(data_dict)
                    all_host_facter_message[file_name] = result_dict
    # return我们最终的数据结果集
    return all_host_facter_message

def register_tags(server_root):
    def join(loader, node):
        seq = loader.construct_sequence(node, deep=True)
        return ''.join([str(i) for i in seq])

    def root(loader, node):
        return server_root + ('' if server_root.endswith('/') else '/')

    def replace(loader, node):
        seq = loader.construct_sequence(node, deep=True)
        return [x.replace(seq[0], seq[1]) for x in seq[2]]

    def prepend(loader, node):
        seq = loader.construct_sequence(node, deep=True)
        if isinstance(seq[1], str):
            return seq[0] + seq[1]
        return [seq[0] + s for s in seq[1]]

    def flatten(loader, node):
        seq = loader.construct_sequence(node, deep=True)
        result = []
        for item in seq:
            if isinstance(item, list):
                result.extend(item)
            else:
                result.append(item)
        return result

    yaml.add_constructor('!join', join)
    yaml.add_constructor('!root', root)
    yaml.add_constructor('!replace', replace)
    yaml.add_constructor('!prepend', prepend)
    yaml.add_constructor('!flatten', flatten)

def get_conf_dict(file):
    mime_type = mimetypes.guess_type(file)[0]

    # Do the type check first, so we don't load huge files that won't be used
    if "xml" in mime_type:
        confile = open(file, "r")
        data = confile.read()
        confile.close()
        return xmltodict(data, "utf-8")
    elif "yaml" in mime_type:
        import yaml

        def custom_str_constructor(loader, node):
            return loader.construct_scalar(node).encode("utf-8")

        yaml.add_constructor(u"tag:yaml.org,2002:str", custom_str_constructor)
        confile = open(file, "r")
        data = confile.read()
        confile.close()
        return yaml.load(data)
    elif "json" in mime_type:
        import json

        confile = open(file, "r")
        data = confile.read()
        confile.close()
        return json.loads(data)

    return False

  def CompileFilter(self, filter_string):
    """Compile a set of ObjectFilters defined in an YAML file."""
    if not os.path.isfile(filter_string):
      raise errors.WrongPlugin((
          'ObjectFilterList requires an YAML file to be passed on, this filter '
          'string is not a file.'))

    yaml.add_constructor('!include', IncludeKeyword,
                         Loader=yaml.loader.SafeLoader)
    results = None

    with open(filter_string, 'rb') as fh:
      try:
        results = yaml.safe_load(fh)
      except (yaml.scanner.ScannerError, IOError) as exception:
        raise errors.WrongPlugin(
            u'Unable to parse YAML file with error: {0:s}.'.format(exception))

    self.filters = []
    results_type = type(results)
    if results_type is dict:
      self._ParseEntry(results)
    elif results_type is list:
      for result in results:
        if type(result) is not dict:
          raise errors.WrongPlugin(
              u'Wrong format of YAML file, entry not a dict ({})'.format(
                  results_type))
        self._ParseEntry(result)
    else:
      raise errors.WrongPlugin(
          u'Wrong format of YAML file, entry not a dict ({})'.format(
              results_type))
    self._filter_expression = filter_string

    def load(self, check_duplicates=True):
        try:
            fp = open(self.path, "r")
        except OSError as e:
            msg = "Cannot open file %s (%s)" % (self.path, e)
            self.add_error(msg)
            return

        if check_duplicates:
            yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, no_duplicates_constructor)
        else:
            yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, constructor)

        try:
            lines = fp.readlines()
            self._contents = yaml.load("".join(lines))

        except (ConstructorError, OurYamlException) as e:
            msg = "Found issues in file %s\n%s" % (self.path, e)
            self.add_warning(msg)
            self.load(check_duplicates=False)

        except (TypeError, ValueError) as e:
            msg = "Cannot parse file %s\n%s" % (self.path, e)
            self.add_error(msg)
            return

        finally:
            fp.close()

        self.is_loaded = True