def _read_tree_config(self):
        """Reads an in-tree config file"""
        dirs = self.query_abs_dirs()
        test_install_dir = dirs.get('abs_test_install_dir',
                                    os.path.join(dirs['abs_work_dir'], 'tests'))

        if 'in_tree_config' in self.config:
            rel_tree_config_path = self.config['in_tree_config']
            tree_config_path = os.path.join(test_install_dir, rel_tree_config_path)

            if not os.path.isfile(tree_config_path):
                self.fatal("The in-tree configuration file '%s' does not exist!"
                           "It must be added to '%s'. See bug 1035551 for more details." %
                           (tree_config_path, os.path.join('gecko', 'testing', rel_tree_config_path)))

            try:
                self.tree_config.update(parse_config_file(tree_config_path))
            except:
                msg = "There was a problem parsing the in-tree configuration file '%s'!" % \
                      os.path.join('gecko', 'testing', rel_tree_config_path)
                self.exception(message=msg, level=FATAL)

            self.dump_config(file_path=os.path.join(dirs['abs_log_dir'], 'treeconfig.json'),
                             config=self.tree_config)

        if (self.buildbot_config and 'properties' in self.buildbot_config and
            self.buildbot_config['properties'].get('branch') == 'try'):
            try_config_path = os.path.join(test_install_dir, 'config', 'mozharness',
                                           'try_arguments.py')
            known_try_arguments = parse_config_file(try_config_path)
            self.set_extra_try_arguments(known_try_arguments)

        self.tree_config.lock()

    def parse_locales_file(self, locales_file):
        locales = []
        c = self.config
        platform = c.get("locales_platform", None)
        ignore_locales = c.get("ignore_locales", None)

        if locales_file.endswith('json'):
            locales_json = parse_config_file(locales_file)
            self.locale_dict = {}
            for locale in locales_json.keys():
                if platform and platform not in locales_json[locale]['platforms']:
                    continue
                locales.append(locale)
                self.locale_dict[locale] = locales_json[locale]['revision']
        else:
            fh = open(locales_file)
            locales = fh.read().split()
            fh.close()
        if ignore_locales:
            for locale in ignore_locales:
                if locale in locales:
                    self.debug("Ignoring locale %s." % locale)
                    locales.remove(locale)
        if locales:
            self.locales = locales
            return self.locales

 def test_dump_config_hierarchy_matches_self_config(self):
     try:
         ######
         # we need temp_cfg because self.s will be gcollected (NoneType) by
         # the time we get to SystemExit exception
         # temp_cfg will differ from self.s.config because of
         # 'dump_config_hierarchy'. we have to make a deepcopy because
         # config is a locked dict
         temp_s = script.BaseScript(
             initial_config_file='test/test.json',
             option_args=['--cfg', 'test/test_override.py,test/test_override2.py'],
         )
         from copy import deepcopy
         temp_cfg = deepcopy(temp_s.config)
         temp_cfg.update({'dump_config_hierarchy': True})
         ######
         self.s = script.BaseScript(
             initial_config_file='test/test.json',
             option_args=['--cfg', 'test/test_override.py,test/test_override2.py'],
             config={'dump_config_hierarchy': True}
         )
     except SystemExit:
         local_cfg_files = parse_config_file('test_logs/localconfigfiles.json')
         # finally let's just make sure that all the items added up, equals
         # what we started with: self.config
         target_cfg = {}
         for cfg_file in local_cfg_files:
             target_cfg.update(local_cfg_files[cfg_file])
         self.assertEqual(
             target_cfg, temp_cfg,
             msg="all of the items (combined) in each cfg file dumped via "
                 "--dump-config-hierarchy does not equal self.config "
         )

 def test_dump_config_equals_self_config(self):
     try:
         ######
         # we need temp_cfg because self.s will be gcollected (NoneType) by
         # the time we get to SystemExit exception
         # temp_cfg will differ from self.s.config because of
         # 'dump_config_hierarchy'. we have to make a deepcopy because
         # config is a locked dict
         temp_s = script.BaseScript(
             initial_config_file='test/test.json',
             option_args=['--cfg', 'test/test_override.py,test/test_override2.py'],
         )
         from copy import deepcopy
         temp_cfg = deepcopy(temp_s.config)
         temp_cfg.update({'dump_config': True})
         ######
         self.s = script.BaseScript(
             initial_config_file='test/test.json',
             option_args=['--cfg', 'test/test_override.py,test/test_override2.py'],
             config={'dump_config': True}
         )
     except SystemExit:
         target_cfg = parse_config_file('test_logs/localconfig.json')
         self.assertEqual(
             target_cfg, temp_cfg,
             msg="all of the items (combined) in each cfg file dumped via "
                 "--dump-config does not equal self.config "
         )

    def _read_tree_config(self):
        """Reads an in-tree config file"""
        dirs = self.query_abs_dirs()
        test_install_dir = dirs.get('abs_test_install_dir',
                                    os.path.join(dirs['abs_work_dir'], 'tests'))

        if 'in_tree_config' in self.config:
            rel_tree_config_path = self.config['in_tree_config']
            tree_config_path = os.path.join(test_install_dir, rel_tree_config_path)

            if not os.path.isfile(tree_config_path):
                self.fatal("The in-tree configuration file '%s' does not exist!"
                           "It must be added to '%s'. See bug 981030 for more details." %
                           (tree_config_path, os.path.join('gecko', 'testing', rel_tree_config_path)))

            try:
                self.tree_config.update(parse_config_file(tree_config_path))
            except:
                msg = "There was a problem parsing the in-tree configuration file '%s'!" % \
                      os.path.join('gecko', 'testing', rel_tree_config_path)
                self.exception(message=msg, level=FATAL)

            self.dump_config(file_path=os.path.join(dirs['abs_log_dir'], 'treeconfig.json'),
                             config=self.tree_config)
        self.tree_config.lock()

 def query_release_config(self):
     if self.release_config:
         return self.release_config
     c = self.config
     dirs = self.query_abs_dirs()
     if c.get("release_config_file"):
         self.info("Getting release config from %s..." % c["release_config_file"])
         rc = None
         try:
             rc = parse_config_file(
                 os.path.join(dirs['abs_work_dir'],
                              c["release_config_file"]),
                 config_dict_name="releaseConfig"
             )
         except IOError:
             self.fatal("Release config file %s not found!" % c["release_config_file"])
         except RuntimeError:
             self.fatal("Invalid release config file %s!" % c["release_config_file"])
         self.release_config['version'] = rc['version']
         self.release_config['buildnum'] = rc['buildNumber']
         self.release_config['ftp_server'] = rc['stagingServer']
         self.release_config['ftp_user'] = c.get('ftp_user', rc['hgUsername'])
         self.release_config['ftp_ssh_key'] = c.get('ftp_ssh_key', rc['hgSshKey'])
         self.release_config['release_channel'] = rc['releaseChannel']
     else:
         self.info("No release config file; using default config.")
         for key in ('version', 'buildnum',
                     'ftp_server', 'ftp_user', 'ftp_ssh_key'):
             self.release_config[key] = c[key]
     self.info("Release config:\n%s" % self.release_config)
     return self.release_config

 def read_buildbot_config(self):
     c = self.config
     if not c.get("buildbot_json_path"):
         # If we need to fail out, add postflight_read_buildbot_config()
         self.info("buildbot_json_path is not set.  Skipping...")
     else:
         # TODO try/except?
         self.buildbot_config = parse_config_file(c["buildbot_json_path"])
         self.info(pprint.pformat(self.buildbot_config))

 def query_talos_json_config(self):
     """Return the talos json config."""
     if self.talos_json_config:
         return self.talos_json_config
     if not self.talos_json:
         self.talos_json = os.path.join(self.talos_path, 'talos.json')
     self.talos_json_config = parse_config_file(self.talos_json)
     self.info(pprint.pformat(self.talos_json_config))
     return self.talos_json_config

 def read_buildbot_config(self):
     c = self.config
     if not c.get("buildbot_json_path"):
         # If we need to fail out, add postflight_read_buildbot_config()
         self.info("buildbot_json_path is not set.  Skipping...")
     else:
         # TODO try/except?
         self.buildbot_config = parse_config_file(c['buildbot_json_path'])
         self.info(json.dumps(self.buildbot_config, indent=4))

    def query_talos_json_config(self):
        if self.talos_json_config:
            return self.talos_json_config

        dirs = self.query_abs_dirs()
        self.talos_json = self.download_file(self.talos_json_url,
                                             parent_dir=dirs['abs_talosdata_dir'],
                                             error_level=FATAL)
        self.talos_json_config = parse_config_file(self.talos_json)
        self.info(pprint.pformat(self.talos_json_config))

        return self.talos_json_config

    def _read_tree_config(self):
        """Reads an in-tree config file"""
        dirs = self.query_abs_dirs()
        test_install_dir = dirs.get('abs_test_install_dir',
                                    os.path.join(dirs['abs_work_dir'], 'tests'))
        tree_config_path = os.path.join(test_install_dir, 'config', 'mozharness_config.py')

        if os.path.isfile(tree_config_path):
            self.tree_config.update(parse_config_file(tree_config_path))
            self.dump_config(file_path=os.path.join(dirs['abs_log_dir'], 'treeconfig.json'),
                             config=self.tree_config)
        self.tree_config.lock()

 def query_talos_json_config(self):
     """Return the talos json config; download and read from the
     talos_json_url if need be."""
     if self.talos_json_config:
         return self.talos_json_config
     if not self.talos_json:
         talos_json_url = self.query_talos_json_url()
         if not talos_json_url:
             self.fatal("Can't download talos_json without a talos_json_url!")
         self.download_talos_json()
     self.talos_json_config = parse_config_file(self.talos_json)
     self.info(pprint.pformat(self.talos_json_config))
     return self.talos_json_config

    def _read_tree_config(self):
        """Reads an in-tree config file"""
        dirs = self.query_abs_dirs()
        test_install_dir = dirs.get('abs_test_install_dir',
                                    os.path.join(dirs['abs_work_dir'], 'tests'))

        if 'in_tree_config' in self.config:
            rel_tree_config_path = self.config['in_tree_config']
            tree_config_path = os.path.join(test_install_dir, rel_tree_config_path)

            if not os.path.isfile(tree_config_path):
                self.fatal("The in-tree configuration file '%s' does not exist!"
                           "It must be added to '%s'. See bug 1035551 for more details." %
                           (tree_config_path, os.path.join('gecko', 'testing', rel_tree_config_path)))

            try:
                self.tree_config.update(parse_config_file(tree_config_path))
            except:
                msg = "There was a problem parsing the in-tree configuration file '%s'!" % \
                      os.path.join('gecko', 'testing', rel_tree_config_path)
                self.exception(message=msg, level=FATAL)

            self.dump_config(file_path=os.path.join(dirs['abs_log_dir'], 'treeconfig.json'),
                             config=self.tree_config)

        if (self.buildbot_config and 'properties' in self.buildbot_config and
            self.buildbot_config['properties'].get('branch') == 'try'):
            try_config_path = os.path.join(test_install_dir, 'config', 'mozharness',
                                           'try_arguments.py')
            known_try_arguments = parse_config_file(try_config_path)
            comments = self.buildbot_config['sourcestamp']['changes'][-1]['comments']
            if not comments and 'try_syntax' in self.buildbot_config['properties']:
                # If we don't find try syntax in the usual place, check for it in an
                # alternate property available to tools using self-serve.
                comments = self.buildbot_config['properties']['try_syntax']
            self.parse_extra_try_arguments(comments, known_try_arguments)

        self.tree_config.lock()

    def checkout_gaia_l10n(self):
        if not self.config.get('gaia_languages_file'):
            self.info('Skipping checkout_gaia_l10n because no gaia language file was specified.')
            return

        l10n_config = self.load_goanna_config().get('gaia', {}).get('l10n')
        if not l10n_config:
            self.fatal("gaia.l10n is required in the goanna config when --gaia-languages-file is specified.")

        abs_work_dir = self.query_abs_dirs()['abs_work_dir']
        languages_file = os.path.join(abs_work_dir, 'gaia', self.config['gaia_languages_file'])
        l10n_base_dir = self.query_abs_dirs()['gaia_l10n_base_dir']

        self.pull_gaia_locale_source(l10n_config, parse_config_file(languages_file).keys(), l10n_base_dir)

    def checkout_gaia_l10n(self):
        if not self.config.get("gaia_languages_file"):
            self.info("Skipping checkout_gaia_l10n because no gaia language file was specified.")
            return

        l10n_config = self.load_gecko_config().get("gaia", {}).get("l10n")
        if not l10n_config:
            self.fatal("gaia.l10n is required in the gecko config when --gaia-languages-file is specified.")

        abs_work_dir = self.query_abs_dirs()["abs_work_dir"]
        languages_file = os.path.join(abs_work_dir, "gaia", self.config["gaia_languages_file"])
        l10n_base_dir = self.query_abs_dirs()["gaia_l10n_base_dir"]

        self.pull_gaia_locale_source(l10n_config, parse_config_file(languages_file).keys(), l10n_base_dir)

 def test_dump_config_hierarchy_valid_files_len(self):
     try:
         self.s = script.BaseScript(
             initial_config_file='test/test.json',
             option_args=['--cfg', 'test/test_override.py,test/test_override2.py'],
             config={'dump_config_hierarchy': True}
         )
     except SystemExit:
         local_cfg_files = parse_config_file('test_logs/localconfigfiles.json')
         # first let's see if the correct number of config files were
         # realized
         self.assertEqual(
             len(local_cfg_files), 4,
             msg="--dump-config-hierarchy dumped wrong number of config files"
         )

 def read_buildbot_config(self):
     c = self.config
     if not c.get("buildbot_json_path"):
         # If we need to fail out, add postflight_read_buildbot_config()
         self.info("buildbot_json_path is not set.  Skipping...")
     else:
         # TODO try/except?
         self.buildbot_config = parse_config_file(c['buildbot_json_path'])
         buildbot_properties = copy.deepcopy(self.buildbot_config.get('properties', {}))
         if 'commit_titles' in buildbot_properties:
             # Remove the commit messages since they can cause false positives with
             # Treeherder log parsers. Eg: "Bug X - Fix TEST-UNEPXECTED-FAIL ...".
             del buildbot_properties['commit_titles']
         self.info("Using buildbot properties:")
         self.info(json.dumps(buildbot_properties, indent=4))

    def parse_locales_file(self, locales_file):
        locales = []
        c = self.config
        platform = c.get("locales_platform", None)

        if locales_file.endswith('json'):
            locales_json = parse_config_file(locales_file)
            self.locale_dict = {}
            for locale in locales_json.keys():
                if platform and platform not in locales_json[locale]['platforms']:
                    continue
                locales.append(locale)
                self.locale_dict[locale] = locales_json[locale]['revision']
        else:
            locales = self.read_from_file(locales_file).split()
        return locales

 def test_dump_config_hierarchy_keys_unique_and_valid(self):
     try:
         self.s = script.BaseScript(
             initial_config_file='test/test.json',
             option_args=['--cfg', 'test/test_override.py,test/test_override2.py'],
             config={'dump_config_hierarchy': True}
         )
     except SystemExit:
         local_cfg_files = parse_config_file('test_logs/localconfigfiles.json')
         # now let's see if only unique items were added from each config
         t_override = local_cfg_files.get('test/test_override.py', {})
         self.assertTrue(
             t_override.get('keep_string') == "don't change me" and len(t_override.keys()) == 1,
             msg="--dump-config-hierarchy dumped wrong keys/value for "
                 "`test/test_override.py`. There should only be one "
                 "item and it should be unique to all the other "
                 "items in test_log/localconfigfiles.json."
         )

    def _update_build_variant(self, rw_config, variant='artifact'):
        """ Intended for use in _pre_config_lock """
        c = self.config
        variant_cfg_path, _ = BuildOptionParser.find_variant_cfg_path(
            '--custom-build-variant-cfg',
            variant,
            rw_config.config_parser
        )
        if not variant_cfg_path:
            self.fatal('Could not find appropriate config file for variant %s' % variant)
        # Update other parts of config to keep dump-config accurate
        # Only dump-config is affected because most config info is set during
        # initial parsing
        variant_cfg_dict = parse_config_file(variant_cfg_path)
        rw_config.all_cfg_files_and_dicts.append((variant_cfg_path, variant_cfg_dict))
        c.update({
            'build_variant': variant,
            'config_files': c['config_files'] + [variant_cfg_path]
        })

        self.info("Updating self.config with the following from {}:".format(variant_cfg_path))
        self.info(pprint.pformat(variant_cfg_dict))
        c.update(variant_cfg_dict)
        c['forced_artifact_build'] = True
        # Bug 1231320 adds MOZHARNESS_ACTIONS in TaskCluster tasks to override default_actions
        # We don't want that when forcing an artifact build.
        if rw_config.volatile_config['actions']:
            self.info("Updating volatile_config to include default_actions "
                      "from {}.".format(variant_cfg_path))
            # add default actions in correct order
            combined_actions = []
            for a in rw_config.all_actions:
                if a in c['default_actions'] or a in rw_config.volatile_config['actions']:
                    combined_actions.append(a)
            rw_config.volatile_config['actions'] = combined_actions
            self.info("Actions in volatile_config are now: {}".format(
                rw_config.volatile_config['actions'])
            )
        # replace rw_config as well to set actions as in BaseScript
        rw_config.set_config(c, overwrite=True)
        rw_config.update_actions()
        self.actions = tuple(rw_config.actions)
        self.all_actions = tuple(rw_config.all_actions)