def test_flatten_metrics_definition(self):
        """Metrics definitions are flattened to a dot-separated list"""
        yaml = '''
- rps
- latency:
  - p50
  - p95
  - p99.9
  - 1
'''
        metrics = MetricsConfig(load_yaml(yaml))
        expected = [
            'latency.1',
            'latency.p50',
            'latency.p95',
            'latency.p99_9',
            'rps',
        ]
        self.assertListEqual(expected, metrics.names)

        yaml = '''
- rps
- latency:
  - nesting:
    - some:
      - more
    - else
'''
        metrics = MetricsConfig(load_yaml(yaml))
        expected = ['latency.nesting.else', 'latency.nesting.some.more', 'rps']
        self.assertListEqual(expected, metrics.names)

def check_attributes_yml(vctx, repo, path, tree):
    try:
        _, attr_blob_sha = tree[".attributes.yml"]
    except KeyError:
        # no .attributes.yml here
        pass
    else:
        from relate.utils import dict_to_struct
        from yaml import load as load_yaml

        att_yml = dict_to_struct(load_yaml(repo[attr_blob_sha].data))

        loc = path + "/" + ".attributes.yml"
        validate_struct(vctx, loc, att_yml, required_attrs=[], allowed_attrs=[("public", list), ("in_exam", list)])

        for access_kind in ["public", "in_exam"]:
            if hasattr(att_yml, access_kind):
                for i, l in enumerate(att_yml.public):
                    if not isinstance(l, six.string_types):
                        raise ValidationError("%s: entry %d in '%s' is not a string" % (loc, i + 1, access_kind))

    import stat

    for entry in tree.items():
        if stat.S_ISDIR(entry.mode):
            _, blob_sha = tree[entry.path]
            subtree = repo[blob_sha]
            check_attributes_yml(vctx, repo, path + "/" + entry.path.decode("utf-8"), subtree)

def get_raw_yaml_from_repo(repo, full_name, commit_sha):
    # type: (Repo_ish, Text, bytes) -> Any
    """Return decoded YAML data structure from
    the given file in *repo* at *commit_sha*.

    :arg commit_sha: A byte string containing the commit hash
    """

    from six.moves.urllib.parse import quote_plus
    cache_key = "%RAW%%2".join((
        CACHE_KEY_ROOT,
        quote_plus(repo.controldir()), quote_plus(full_name), commit_sha.decode(),
        ))

    import django.core.cache as cache
    def_cache = cache.caches["default"]

    result = None  # type: Optional[Any]
    # Memcache is apparently limited to 250 characters.
    if len(cache_key) < 240:
        result = def_cache.get(cache_key)
    if result is not None:
        return result

    yaml_str = expand_yaml_macros(
                repo, commit_sha,
                get_repo_blob(repo, full_name, commit_sha,
                    allow_tree=False).data)

    result = load_yaml(yaml_str)  # type: ignore

    def_cache.add(cache_key, result, None)

    return result

    def post(self):
        form = ConfigurationConverterForm()

        if not form.validate_on_submit():
            return render_template("superbvote/convert.html", form=form)

        try:
            yaml = load_yaml(form.yaml_file.data)
        except YAMLError:
            flash(
                Markup('A syntax error was detected. You may want to use '
                       '<a href="http://yaml-online-parser.appspot.com/">this tool</a> '
                       'to determine the problem.'),
                "formerror")
            return render_template("superbvote/convert.html", form=form)

        if not isinstance(yaml, dict):
            flash("This YAML file does not represent a dictionary (mapping).", "formerror")
            return render_template("superbvote/convert.html", form=form)

        try:
            result = process_configuration(yaml)
        except Exception as e:
            flash("This YAML file does not look like a GAListener configuration.", "formerror")
            return render_template("superbvote/convert.html", form=form)

        return make_response((dump(result), 200, {"Content-Type": "application/yaml",
                                                  "Content-Disposition": "attachment; filename=superbvote_config.yml"}))

def get_yaml_from_repo(repo, full_name, commit_sha, cached=True):
    """Return decoded, struct-ified YAML data structure from
    the given file in *repo* at *commit_sha*.

    See :class:`relate.utils.Struct` for more on
    struct-ification.
    """

    if cached:
        cache_key = "%%%2".join((repo.controldir(), full_name, commit_sha))

        import django.core.cache as cache

        def_cache = cache.caches["default"]
        result = def_cache.get(cache_key)
        if result is not None:
            return result

    result = dict_to_struct(
        load_yaml(expand_yaml_macros(repo, commit_sha, get_repo_blob(repo, full_name, commit_sha).data))
    )

    if cached:
        def_cache.add(cache_key, result, None)

    return result

def get_raw_yaml_from_repo(repo, full_name, commit_sha):
    """Return decoded YAML data structure from
    the given file in *repo* at *commit_sha*.

    :arg commit_sha: A byte string containing the commit hash
    """

    from six.moves.urllib.parse import quote_plus
    cache_key = "%RAW%%2".join((
        quote_plus(repo.controldir()), quote_plus(full_name), commit_sha.decode()))

    import django.core.cache as cache
    def_cache = cache.caches["default"]
    result = None
    # Memcache is apparently limited to 250 characters.
    if len(cache_key) < 240:
        result = def_cache.get(cache_key)
    if result is not None:
        return result

    result = load_yaml(
            expand_yaml_macros(
                repo, commit_sha,
                get_repo_blob(repo, full_name, commit_sha).data))

    def_cache.add(cache_key, result, None)

    return result

def check_attributes_yml(vctx, repo, path, tree):
    try:
        _, attr_blob_sha = tree[".attributes.yml"]
    except KeyError:
        # no .attributes.yml here
        pass
    else:
        from relate.utils import dict_to_struct
        from yaml import load as load_yaml

        att_yml = dict_to_struct(load_yaml(repo[attr_blob_sha].data))

        loc = path + "/" + ".attributes.yml"
        validate_struct(
                vctx, loc, att_yml,
                required_attrs=[],
                allowed_attrs=[
                    ("public", list),
                ])

        if hasattr(att_yml, "public"):
            for i, l in enumerate(att_yml.public):
                if not isinstance(l, (str, unicode)):
                    raise ValidationError(
                            "%s: entry %d in 'public' is not a string"
                            % (loc, i+1))

    import stat
    for entry in tree.items():
        if stat.S_ISDIR(entry.mode):
            _, blob_sha = tree[entry.path]
            subtree = repo[blob_sha]
            check_attributes_yml(vctx, repo, path+"/"+entry.path, subtree)

def get_yaml_from_repo(repo, full_name, commit_sha, cached=True):
    """Return decoded, struct-ified YAML data structure from
    the given file in *repo* at *commit_sha*.

    See :class:`relate.utils.Struct` for more on
    struct-ification.
    """

    if cached:
        from six.moves.urllib.parse import quote_plus
        cache_key = "%%%2".join(
                (quote_plus(repo.controldir()), quote_plus(full_name),
                    commit_sha.decode()))

        import django.core.cache as cache
        def_cache = cache.caches["default"]
        result = None
        # Memcache is apparently limited to 250 characters.
        if len(cache_key) < 240:
            result = def_cache.get(cache_key)
        if result is not None:
            return result

    expanded = expand_yaml_macros(
            repo, commit_sha,
            get_repo_blob(repo, full_name, commit_sha).data)

    result = dict_to_struct(load_yaml(expanded))

    if cached:
        def_cache.add(cache_key, result, None)

    return result

def load_config(app_name):
    global config

    search_filenames = [
        os.path.expanduser("~/.fundraising/%s.yaml" % app_name),
        os.path.expanduser("~/.%s.yaml" % app_name),
        # FIXME: relative path fail
        os.path.dirname(__file__) + "/../%s/config.yaml" % app_name,
        "/etc/fundraising/%s.yaml" % app_name,
        "/etc/%s.yaml" % app_name,
        # FIXME: relative path fail
        os.path.dirname(__file__) + "/../%s/%s.yaml" % (app_name, app_name,)
    ]
    # TODO: if getops.get(--config/-f): search_filenames.append

    for filename in search_filenames:
        if not os.path.exists(filename):
            continue

        config = DictAsAttrDict(load_yaml(file(filename, 'r')))
        log.info("Loaded config from {path}.".format(path=filename))

        config.app_name = app_name

        return

    raise Exception("No config found, searched " + ", ".join(search_filenames))

def load_config(app_name):
    global _config

    search_filenames = [
        os.path.expanduser("~/.fundraising/%s.yaml" % app_name),
        os.path.expanduser("~/.%s.yaml" % app_name),
        # FIXME: relative path fail
        os.path.dirname(__file__) + "/../%s/config.yaml" % app_name,
        "/etc/fundraising/%s.yaml" % app_name,
        "/etc/%s.yaml" % app_name,
        # FIXME: relative path fail
        os.path.dirname(__file__) + "/../%s/%s.yaml" % (app_name, app_name,)
    ]
    # TODO: if getops.get(--config/-f): search_filenames.append

    for filename in search_filenames:
        if not os.path.exists(filename):
            continue

        _config = DictAsAttrDict(load_yaml(file(filename, 'r')))
        log.info("Loaded config from {path}.".format(path=filename))

        _config.app_name = app_name

        # TODO: Move up a level, entry point should directly call logging
        # configuration.
        process.log.setup_logging()

        return _config

    raise Exception("No config found, searched " + ", ".join(search_filenames))

    def post(self):
        form = YamlCheckerForm()

        if not form.validate_on_submit():
            return render_template("yamlchecker/main.html", form=form)

        if form.type.data == "bungeecord":
            checker = BungeeCordConfigChecker()
        elif form.type.data == "redisbungee":
            checker = RedisBungeeConfigChecker()
        else:
            form.type.errors.append("This is an invalid configuration type.")
            return render_template("yamlchecker/main.html", form=form)

        try:
            yaml = load_yaml(form.yaml_file.data)
        except YAMLError:
            flash(
                Markup('A syntax error was detected. You may want to use '
                       '<a href="http://yaml-online-parser.appspot.com/">this tool</a> '
                       'to determine the problem.'),
                "formerror")
            return render_template("yamlchecker/main.html", form=form)

        if not isinstance(yaml, dict):
            flash("This YAML file does not represent a dictionary (mapping).", "formerror")
            return render_template("yamlchecker/main.html", form=form)

        for message in checker.check_config(yaml):
            flash(message['message'], message['class'])

        return render_template("yamlchecker/main.html", validated=len(get_flashed_messages()) == 0,
                               form=form)

    def run(self, spec):
        """Execute the various tasks given in the spec list."""

        try:

            if output.debug:
                names = ", ".join(info[0] for info in spec)
                print("Tasks to run: %s" % names)

            call_hooks('commands.before', self.tasks, spec)

            # Initialise the default stage if none are given as the first task.
            if 'stages' in env:
                if spec[0][0] not in env.stages:
                    self.execute_task(env.stages[0], (), {}, None)
                else:
                    self.execute_task(*spec.pop(0))

            # Load the config YAML file if specified.
            if env.config_file:
                config_path = realpath(expanduser(env.config_file))
                config_path = join(self.directory, config_path)
                config_file = open(config_path, 'rb')
                config = load_yaml(config_file.read())
                if not config:
                    env.config = AttrDict()
                elif not isinstance(config, dict):
                    abort("Invalid config file found at %s" % config_path)
                else:
                    env.config = AttrDict(config)
                config_file.close()

            call_hooks('config.loaded')

            # Execute the tasks in order.
            for info in spec:
                self.execute_task(*info)

            if output.status:
                msg = "\nDone."
                if env.colors:
                    msg = env.color_settings['finish'](msg)
                print(msg)

        except SystemExit:
            raise
        except KeyboardInterrupt:
            if output.status:
                msg = "\nStopped."
                if env.colors:
                    msg = env.color_settings['finish'](msg)
                print >> sys.stderr, msg
            sys.exit(1)
        except:
            sys.excepthook(*sys.exc_info())
            sys.exit(1)
        finally:
            call_hooks('commands.after')
            disconnect_all()

def load_yaml_config(fpath):
    verify_file(fpath, is_critical=True)
    try:
        dic = load_yaml(open(fpath))
    except Exception:
        err(format_exc())
        critical('Could not parse bcbio YAML ' + fpath)
    else:
        return dic

 def _process_yaml(self, yaml_string):
     yaml_motifs = load_yaml(yaml_string)
     for entry in yaml_motifs:
         polys = list(Polymorphism(int(k),0,v) for k,v in entry['polymorphisms'].iteritems())
         self[entry['id']] = Motif(id=entry['id'],
                                   label=entry['label'],
                                   sources=entry['source'],
                                   polymorphisms=polys)
         self.__sources.update(self[entry['id']].sources)

    def __init__(self, yaml_file, root_path, defaults=None):
        super(Config, self).__init__(root_path, defaults)

        if not os.path.exists(yaml_file):
            raise ConfigError('Configuration file "%s" does not exists.' % yaml_file)

        with open(yaml_file, 'r') as f:
            conf_data = f.read()

        self.update(load_yaml(conf_data))

def read_config():
    for directory, filename in product(CONFIG_DIRECTORIES, CONFIG_FILES):
        try:
            config = join(directory, filename)
            _LOGGER.debug("checking for config file %s", config)
            with open(config) as config:
                return list(load_yaml(config))
        except (IOError, OSError):
            continue
    return {}

def override(settings, yaml=None, env=None):
    """
    :param dict settings: settings dict to be updated; usually it's ``globals()``
    :param yaml: path to YAML file
    :type yaml: str or FileIO
    :param str env: prefix for environment variables
    """
    if yaml is not None:
        if hasattr(yaml, 'read'):
            settings.update(load_yaml(yaml.read()))
        else:
            if os.path.exists(yaml):
                with open(yaml) as f:
                    settings.update(load_yaml(f.read()))

    if env is not None:
        for k, v in os.environ.items():
            if k.startswith(env):
                settings[k[len(env):]] = load_yaml(v)

    def load_yaml_file(self, path):
        """ Load the data in the file as YAML, then deserialize into the model
        with the schema

        :param path: Path to the YAML file to load
        :type path: py.path.local
        """
        with path.open('r') as handle:
            data = load_yaml(handle)

        self.set_all(**self.SCHEMA.load(data).data)

 def _load_data(self):
     if self.file is None:
         data = get_data('data', 'languages.yaml')
     else:
         data = self.file.read()
     data = load_yaml(data)
     base_data = data.pop('base', {'skip': []})
     known_languages = {}
     for shortname, language_info in six.iteritems(data):
         self._update_language_info_with_base_info(language_info, base_data)
         language = Language(shortname, language_info)
         if language.validate_info():
             known_languages[shortname] = language
     self._data = known_languages

    def __init__(self, **kwargs):
        """
        Settings are now loaded using the data/settings.yaml file.
        """

        data = get_data('data', 'settings.yaml')
        data = load_yaml(data)
        settings_data = data.pop('settings', {})

        for datum in settings_data:
            setattr(self, datum, settings_data[datum])

        for key in kwargs:
            setattr(self, key, kwargs[key])