def test_update_engine_settings(self):
     
     self.assertRaises(TankError, self.env.update_engine_settings, "bad_engine", {}, {})
     
     # get raw environment before
     env_file = os.path.join(self.project_config, "env", "test.yml")
     fh = open(env_file)
     env_before = yaml.load(fh)
     fh.close()
     prev_settings = self.env.get_engine_settings("test_engine")
     
     self.env.update_engine_settings("test_engine", {"foo":"bar"}, {"type":"dev", "path":"foo"})
     
     # get raw environment after
     env_file = os.path.join(self.project_config, "env", "test.yml")
     fh = open(env_file)
     env_after = yaml.load(fh)
     fh.close()
     
     # ensure that disk was updated
     self.assertNotEqual(env_after, env_before)
     env_before["engines"]["test_engine"]["foo"] = "bar"
     env_before["engines"]["test_engine"]["location"] = {"type":"dev", "path":"foo"}
     self.assertEqual(env_after, env_before)
     
     # ensure memory was updated
     new_settings = self.env.get_engine_settings("test_engine")
     prev_settings.update({"foo":"bar"})
     self.assertEqual(new_settings, prev_settings)
     
     desc_after = self.env.get_engine_descriptor("test_engine")
     self.assertEqual(desc_after.get_location(), {"type":"dev", "path":"foo"})

    def test_add_app(self):

        self.assertRaises(TankError, self.env.create_app_settings, "test_engine", "test_app")
        self.assertRaises(TankError, self.env.create_app_settings, "unknown_engine", "test_app")

        # get raw environment before
        env_file = os.path.join(self.project_config, "env", "test.yml")
        with open(env_file) as fh:
            env_before = yaml.load(fh)

        self.env.create_app_settings("test_engine", "new_app")

        # get raw environment after
        env_file = os.path.join(self.project_config, "env", "test.yml")
        with open(env_file) as fh:
            env_after = yaml.load(fh)

        # ensure that disk was updated
        self.assertNotEqual(env_after, env_before)
        env_before["engines"]["test_engine"]["apps"]["new_app"] = {}
        env_before["engines"]["test_engine"]["apps"]["new_app"]["location"] = {}
        self.assertEqual(env_after, env_before)

        # ensure memory was updated
        cfg_after = self.env.get_app_settings("test_engine", "new_app")
        self.assertEqual(cfg_after, {})

    def setUp(self):
        super(TestEnvironment, self).setUp()
        self.setup_fixtures()

        self.test_env = "test"
        self.test_engine = "test_engine"

        # create env object
        self.env = self.tk.pipeline_configuration.get_environment(self.test_env)

        # get raw environment
        env_file = os.path.join(self.project_config, "env", "test.yml")
        fh = open(env_file)
        self.raw_env_data = yaml.load(fh)
        fh.close()

        # get raw app metadata
        app_md = os.path.join(self.project_config, "bundles", "test_app", "info.yml")
        fh = open(app_md)
        self.raw_app_metadata = yaml.load(fh)
        fh.close()

        # get raw engine metadata
        eng_md = os.path.join(self.project_config, "bundles", "test_engine", "info.yml")
        fh = open(eng_md)
        self.raw_engine_metadata = yaml.load(fh)
        fh.close()

 def _load_nuke_publish_snapshot_comments(self, snapshot_file_path):
     """
     Load old nuke-style snapshot comments if they exist.  These are only
     ever read - all new comments are saved to the new file.
     """
     comments = {}
     try:
         # look for old nuke style path:        
         snapshot_dir = os.path.dirname(snapshot_file_path)
         fields = self._snapshot_template.get_fields(snapshot_file_path)
         
         SNAPSHOT_COMMENTS_FILE = r"%s_comments.yml"
         comments_file_name = SNAPSHOT_COMMENTS_FILE % fields.get("name", "unknown")
         comments_file_path = os.path.join(snapshot_dir, comments_file_name)
         
         comments = {}
         if os.path.exists(comments_file_path):
             raw_comments = yaml.load(open(comments_file_path, "r"))
             for (name, timestamp), comment in raw_comments.iteritems():
                 fields["name"] = name
                 fields["timestamp"] = timestamp
                 snapshot_path = self._snapshot_template.apply_fields(fields)
                 
                 if os.path.exists(snapshot_path):
                     # add comment to dictionary in new style:
                     comments_key = os.path.basename(snapshot_path)
                     comments[comments_key] = {"comment":comment}
     except:
         # it's not critical that this succeeds so just ignore any exceptions
         pass
         
     return comments

 def _get_snapshot_comments(self, snapshot_file_path):
     """
     Return the snapshot comments for the specified file path
     """
     # first, attempt to load old-nuke-publish-style comments:
     comments = self._load_nuke_publish_snapshot_comments(snapshot_file_path)
     
     # now load new style comments:
     comments_file_path = self._get_comments_file_path(snapshot_file_path)
     raw_comments = {}
     if os.path.exists(comments_file_path):
         raw_comments = yaml.load(open(comments_file_path, "r"))
         
     # process raw comments to convert old-style to new if need to:
     for key, value in raw_comments.iteritems():
         if isinstance(value, basestring):
             # old style string
             comments[key] = {"comment":value}
         elif isinstance(value, dict):
             # new style dictionary
             comments[key] = value
         else:
             # value isn't valid!
             pass
             
     # ensure all comments are returned as utf-8 strings rather than
     # unicode - this is due to a previous bug where the snapshot UI
     # would return the comment as unicode!
     for comment_dict in comments.values():
         comment = comment_dict.get("comment")
         if comment and isinstance(comment, unicode):
             comment_dict["comment"] = comment.encode("utf8")
         
     return comments        

    def _get_roots_data(self):
        """
        Returns roots.yml data for this config.
        If no root file can be loaded, {} is returned.

        :returns: Roots data yaml content, usually a dictionary
        """
        self._io_descriptor.ensure_local()

        # get the roots definition
        root_file_path = os.path.join(
            self._io_descriptor.get_path(),
            "core",
            constants.STORAGE_ROOTS_FILE)

        roots_data = {}

        if os.path.exists(root_file_path):
            root_file = open(root_file_path, "r")
            try:
                # if file is empty, initializae with empty dict...
                roots_data = yaml.load(root_file) or {}
            finally:
                root_file.close()

        return roots_data

    def test_project_root_mismatch(self):
        """
        Case that root name specified in projects yml file does not exist in roots file.
        """
        # remove root name from the roots file
        self.setup_multi_root_fixtures()
        self.tk = tank.Tank(self.project_root)
        
        # should be fine
        folder.configuration.FolderConfiguration(self.tk, self.schema_location)

        roots_file = os.path.join(self.tk.pipeline_configuration.get_path(), "config", "core", "schema", "alternate_1.yml")
        
        fh = open(roots_file, "r")
        data = yaml.load(fh)
        fh.close()
        data["root_name"] = "some_bogus_Data"
        fh = open(roots_file, "w")
        fh.write(yaml.dump(data))
        fh.close()

        self.tk = tank.Tank(self.project_root)

        self.assertRaises(TankError,
                          folder.configuration.FolderConfiguration,
                          self.tk,
                          self.schema_location)

def get_pc_registered_location(pipeline_config_root_path):
    """
    Loads the location metadata file from install_location.yml
    This contains a reflection of the paths given in the pc entity.

    Returns the path that has been registered for this pipeline configuration 
    for the current OS.
    This is the path that has been defined in shotgun. It is also the path that is being
    used in the inverse pointer files that exist in each storage.
    
    This is useful when drive letter mappings or symlinks are being used - in these
    cases get_path() may not return the same value as get_registered_location_path().
    
    This may return None if no path has been registered for the current os.
    """
    # now read in the pipeline_configuration.yml file
    cfg_yml = os.path.join(pipeline_config_root_path, "config", "core", "install_location.yml")

    if not os.path.exists(cfg_yml):
        raise TankError("Location metadata file '%s' missing! Please contact support." % cfg_yml)

    fh = open(cfg_yml, "rt")
    try:
        data = yaml.load(fh)
    except Exception, e:
        raise TankError("Looks like a config file is corrupt. Please contact "
                        "support! File: '%s' Error: %s" % (cfg_yml, e))

 def _get_metadata(self):
     """
     Returns the info.yml metadata associated with this descriptor.
     Note that this call involves deep introspection; in order to
     access the metadata we normally need to have the code content
     local, so this method may trigger a remote code fetch if necessary.
     """
     if self.__manifest_data is None:
         # make sure payload exists locally
         if not self.exists_local():
             self.download_local()
             
         # get the metadata
         bundle_root = self.get_path()
     
         file_path = os.path.join(bundle_root, constants.BUNDLE_METADATA_FILE)
     
         if not os.path.exists(file_path):
             raise TankError("Toolkit metadata file '%s' missing." % file_path)
     
         try:
             file_data = open(file_path)
             try:
                 metadata = yaml.load(file_data)
             finally:
                 file_data.close()
         except Exception, exp:
             raise TankError("Cannot load metadata file '%s'. Error: %s" % (file_path, exp))
     
         # cache it
         self.__manifest_data = metadata

    def test_url_cleanup(self):

        # Make sure that if a file has the url saved incorrectly...
        with patch("sgtk.util.shotgun.connection.sanitize_url", wraps=lambda x: x):
            session_cache.set_current_host("https://host.cleaned.up.on.read/")
            # ... then sure we indeed disabled cleanup and that the malformed value was written to disk...
            self.assertEquals("https://host.cleaned.up.on.read/", session_cache.get_current_host())

        # ... and finaly that the value is filtered when being read back from disk.
        self.assertEquals("https://host.cleaned.up.on.read", session_cache.get_current_host())

        # Make sure we're cleaning up the hostname when saving it.
        session_cache.set_current_host("https://host.cleaned.up.on.write/")

        with open(
            os.path.join(
                LocalFileStorageManager.get_global_root(
                    LocalFileStorageManager.CACHE
                ),
                "authentication.yml"
            ),
            "r"
        ) as fh:
            # Let's read the file directly to see if the data was cleaned up.
            data = yaml.load(fh)
            self.assertEqual(
                data["current_host"],
                "https://host.cleaned.up.on.write"
            )

    def get_associated_core_version(self):
        """
        Returns the version string for the core api associated with this config.
        This method is 'forgiving' and in the case no associated core API can be 
        found for this pipeline configuration, None will be returned rather than 
        an exception raised. 
        """
        associated_api_root = self.get_install_location()
        
        info_yml_path = os.path.join(associated_api_root, "core", "info.yml")

        if os.path.exists(info_yml_path):
            try:
                info_fh = open(info_yml_path, "r")
                try:
                    data = yaml.load(info_fh)
                finally:
                    info_fh.close()
                data = data.get("version")
            except:
                data = None
        else:
            data = None

        return data

def get_pc_roots_metadata(pipeline_config_root_path):
    """
    Loads and validates the roots metadata file.
    
    The roots.yml file is a reflection of the local storages setup in Shotgun
    at project setup time and may contain anomalies in the path layout structure.
    
    The roots data will be prepended to paths and used for comparison so it is 
    critical that the paths are on a correct normalized form once they have been 
    loaded into the system.    
    """
    # now read in the roots.yml file
    # this will contain something like
    # {'primary': {'mac_path': '/studio', 'windows_path': None, 'linux_path': '/studio'}}
    roots_yml = os.path.join(pipeline_config_root_path, "config", "core", "roots.yml")

    if not os.path.exists(roots_yml):
        raise TankError("Roots metadata file '%s' missing! Please contact support." % roots_yml)

    fh = open(roots_yml, "rt")
    try:
        data = yaml.load(fh)
    except Exception, e:
        raise TankError("Looks like the roots file is corrupt. Please contact "
                        "support! File: '%s' Error: %s" % (roots_yml, e))

    def _get_metadata(self):
        """
        Loads the pipeline config metadata (the pipeline_configuration.yml) file from disk.

        :param pipeline_config_path: path to a pipeline configuration root folder
        :returns: deserialized content of the file in the form of a dict.
        """

        # now read in the pipeline_configuration.yml file
        cfg_yml = self._get_pipeline_config_file_location()

        if not os.path.exists(cfg_yml):
            raise TankError("Configuration metadata file '%s' missing! "
                            "Please contact support." % cfg_yml)

        fh = open(cfg_yml, "rt")
        try:
            data = yaml.load(fh)
            if data is None:
                raise Exception("File contains no data!")
        except Exception as e:
            raise TankError("Looks like a config file is corrupt. Please contact "
                            "support! File: '%s' Error: %s" % (cfg_yml, e))
        finally:
            fh.close()

        return data

def _get_install_locations(path):
    """
    Given a pipeline configuration OR core location, return paths on all platforms.
    
    :param path: Path to a pipeline configuration on disk.
    :returns: dictionary with keys linux2, darwin and win32
    """
    # basic sanity check
    if not os.path.exists(path):
        raise TankError("The core path '%s' does not exist on disk!" % path)
    
    # for other platforms, read in install_location
    location_file = os.path.join(path, "config", "core", "install_location.yml")
    if not os.path.exists(location_file):
        raise TankError("Cannot find core config file '%s' - please contact support!" % location_file)

    # load the config file
    try:
        open_file = open(location_file)
        try:
            location_data = yaml.load(open_file)
        finally:
            open_file.close()
    except Exception, error:
        raise TankError("Cannot load core config file '%s'. Error: %s" % (location_file, error))

def get_location(sgtk, app_bootstrap):
    """
    Returns a location dictionary for the bundled framework.

    :param sgtk: Handle to the Toolkit API
    :param app_bootstrap: Instance of the application bootstrap.

    :returns: A dictionary with keys and values following the Toolkit location
        convention. Read more at https://support.shotgunsoftware.com/entries/95442678#Code%20Locations
    """
    dev_descriptor = {
        "type": "dev",
        "path": app_bootstrap.get_startup_location_override()
    }
    # If the startup location had been overriden, the descriptor is automatically
    # a dev descriptor.
    if app_bootstrap.get_startup_location_override():
        return dev_descriptor
    # If we are running the bundled startup, it is possible to override it's
    # value on first run in order to trigger the system into pulling from
    # another location that the app_store, namely git. This is done through an
    # environment variable. This is also great for testing app_store updates
    # by pretending you have an older version of the code bundled and need an update.

    # Local import since sgtk is lazily loaded.
    from tank_vendor import yaml
    if app_bootstrap.runs_bundled_startup() and "SGTK_DESKTOP_BUNDLED_DESCRIPTOR" in os.environ:
        try:
            return yaml.load(os.environ["SGTK_DESKTOP_BUNDLED_DESCRIPTOR"])
        except yaml.YAMLError, e:
            raise BundledDescriptorEnvVarError(e)

def _process_includes_r(file_name, data, context):
    """
    Recursively process includes for an environment file.
    
    Algorithm (recursive):
    
    1. Load include data into a big dictionary X
    2. recursively go through the current file and replace any 
       @ref with a dictionary value from X
    
    :param file_name:   The root yml file to process
    :param data:        The contents of the root yml file to process
    :param context:     The current context

    :returns:           A tuple containing the flattened yml data 
                        after all includes have been recursively processed
                        together with a lookup for frameworks to the file 
                        they were loaded from.
    """
    # first build our big fat lookup dict
    include_files = _resolve_includes(file_name, data, context)
    
    lookup_dict = {}
    fw_lookup = {}
    for include_file in include_files:
                
        # path exists, so try to read it
        fh = open(include_file, "r")
        try:
            included_data = yaml.load(fh) or {}
        finally:
            fh.close()
                
        # now resolve this data before proceeding
        included_data, included_fw_lookup = _process_includes_r(include_file, included_data, context)

        # update our big lookup dict with this included data:
        if "frameworks" in included_data and isinstance(included_data["frameworks"], dict):
            # special case handling of frameworks to merge them from the various
            # different included files rather than have frameworks section from
            # one file overwrite the frameworks from previous includes!
            lookup_dict = _resolve_frameworks(included_data, lookup_dict)

            # also, keey track of where the framework has been referenced from:
            for fw_name in included_data["frameworks"].keys():
                fw_lookup[fw_name] = include_file

            del(included_data["frameworks"])

        fw_lookup.update(included_fw_lookup)
        lookup_dict.update(included_data)
    
    # now go through our own data, recursively, and replace any refs.
    # recurse down in dicts and lists
    try:
        data = _resolve_refs_r(lookup_dict, data)
        data = _resolve_frameworks(lookup_dict, data)
    except TankError, e:
        raise TankError("Include error. Could not resolve references for %s: %s" % (file_name, e))

    def run(self, log, args):
        if len(args) != 4:

            log.info("Syntax: move_studio_install current_path linux_path windows_path mac_path")
            log.info("")
            log.info("This command will move the main location of the Toolkit config.")
            log.info("")
            log.info("Specify the current location of your studio install in the first parameter. "
                     "Specify the new location for each platform in the subsequent parameters.")
            log.info("")
            log.info("You typically need to quote your paths, like this:")
            log.info("")
            log.info('> tank move_studio_install /projects/tank /tank_install/studio "p:\\tank_install\\studio" /tank_install/studio')
            log.info("")
            log.info("If you want to leave a platform blank, just just empty quotes:")
            log.info("")
            log.info('> tank move_studio_install "P:\\projects\\tank" "" "p:\\tank_install\\studio" ""')
            raise TankError("Wrong number of parameters!")
        
        current_path = args[0]
        linux_path = args[1]
        windows_path = args[2]
        mac_path = args[3]
        new_paths = {"mac_path": mac_path, 
                     "windows_path": windows_path, 
                     "linux_path": linux_path}
        storage_map = {"linux2": "linux_path", "win32": "windows_path", "darwin": "mac_path" }
        local_target_path = new_paths.get(storage_map[sys.platform])

        # basic checks
        if not os.path.exists(current_path):
            raise TankError("Path '%s' does not exist!" % current_path)
        
        if os.path.exists(local_target_path):
            raise TankError("The path %s already exists on disk!" % local_target_path)
        
        # probe for some key file
        api_file = os.path.join(current_path, "install", "core", "_core_upgrader.py")
        if not os.path.exists(api_file):
            raise TankError("Path '%s' does not look like an Toolkit install!" % current_path)
            
        # make sure this is NOT a PC
        pc_file = os.path.join(current_path, "config", "info.yml")
        if os.path.exists(pc_file):
            raise TankError("Path '%s' does not look like a pipeline configuration. Move it "
                            "using the move_configuration command instead!" % current_path)
                
        # now read in the pipeline_configuration.yml file
        # to find out the locations for all other platforms
        cfg_yml = os.path.join(current_path, "config", "core", "install_location.yml")
        if not os.path.exists(cfg_yml):
            raise TankError("Location metadata file '%s' missing!" % cfg_yml)
        fh = open(cfg_yml, "rt")
        try:
            data = yaml.load(fh)
        except Exception, e:
            raise TankError("Config file %s is invalid: %s" % (cfg_yml, e))

 def __load_data(self, path):
     """
     loads the main data from disk, raw form
     """
     # load the data in 
     try:
         env_file = open(path, "r")
         data = yaml.load(env_file)
     except Exception, exp:
         raise TankError("Could not parse file %s. Error reported: %s" % (path, exp))

 def _populate_cache_item_data(self, item):
     """
     Loads the CacheItem's YAML data from disk.
     """
     path = item.path
     try:
         with open(path, "r") as fh:
             raw_data = yaml.load(fh)
     except IOError:
         raise TankFileDoesNotExistError("File does not exist: %s" % path)
     except Exception, e:
         raise TankError("Could not open file '%s'. Error reported: '%s'" % (path, e))

    def _get_shotgun_yml_content(self, cw):
        """
        Retrieves the content of the shotgun.yml file based on the configuration writer passed in.

        :param cw: Configuration writer used to write the shotgun.yml file.

        :returns: Path to the Shotgun file.
        """
        shotgun_yml_path = os.path.join(cw.path.current_os, "config", "core", "shotgun.yml")
        self.assertTrue(os.path.exists(shotgun_yml_path))
        with open(shotgun_yml_path, "rb") as fh:
            return yaml.load(fh)