def addEnvarsToProfile(self, node):
     """
         Add environment variables (SGE_CELL, ports, etc.) to /etc/profile
     """
     envars = self.exportEnvironmentVars();
     log.debug("sge.addEnvarsToProfile    envars: echo '%s' >> /etc/profile", envars)
     node.ssh.execute("echo '" + envars + "' >> /etc/profile")

	def _probe_peers(self, master, nodes):
		cmd = ""
		log.info("Probing %d nodes" % len(nodes))
		for node in nodes:
			cmd += "/usr/sbin/gluster peer probe %s;" % node.alias
			log.debug(master.ssh.execute(cmd))
		log.debug(master.ssh.execute("/usr/sbin/gluster peer status"))

 def _eval_remove_node(self):
     """
     This function uses the sge stats to decide whether or not to
     remove a node from the cluster.
     """
     qlen = len(self.stat.get_queued_jobs())
     if qlen != 0:
         return
     if not self.has_cluster_stabilized():
         return
     if len(self.stat.hosts) <= self.min_nodes:
         log.info("Not removing nodes: already at or below minimum (%d)"
                  % self.min_nodes)
         return
     max_remove = len(self.stat.hosts) - self.min_nodes
     log.info("Looking for nodes to remove...")
     remove_nodes = self._find_nodes_for_removal(max_remove=max_remove)
     if not remove_nodes:
         log.info("No nodes can be removed at this time")
     for node in remove_nodes:
         if node.update() != "running":
             log.error("Node %s is already dead - not removing" %
                       node.alias)
             continue
         log.warn("Removing %s: %s (%s)" %
                  (node.alias, node.id, node.dns_name))
         try:
             self._cluster.remove_node(node)
             self.__last_cluster_mod_time = datetime.datetime.utcnow()
         except Exception:
             log.error("Failed to remove node %s" % node.alias)
             log.debug(traceback.format_exc())

    def run(self, nodes, master, user, user_shell, volumes):

        #master.ssh.execute(
            #"killall -9 pbs_server; killall -9 pbs_sched; CLEAN_DELAY=0 emerge -C torque; rm -rvf /var/spool/torque; FEATURES=buildpkg emerge -g -j torque",
            #silent=False)
        #import IPython; ipshell = IPython.embed; ipshell(banner1='ipshell')

        # -- configure torque's server and scheduler on the master node
        log.info("Configuring torque server...")
        master.ssh.execute(master_configure_server)

        # -- configure torque's clients on each node and complete the
        # configuration on the master node
        for node in nodes[1:]:
            log.info("Configuring torque node '%s'..." % node.alias)
            node.ssh.execute(node_configure_mom)
            self._add_torque_node_to_master(node, master)

        # -- (re)start services
        log.info("Starting torque services...")
        self._force_deamon_restart(master, 'pbs_server')
        for node in nodes[1:]:
            self._start_torque_node_daemon(node)
        self._force_deamon_restart(master, 'pbs_sched')

        # -- print infos / debug
        log.debug("Torque server information:")
        master.ssh.execute("qmgr -c 'l s'")
        master.ssh.execute("qmgr -c 'p s'")

        log.debug("Torque nodes information:")
        for node in nodes[1:]:
            master.ssh.execute('momctl -h %s -d 2' % node.alias)
        master.ssh.execute("qnodes")

 def alias(self):
     """
     Fetches the node's alias stored in a tag from either the instance
     or the instance's parent spot request. If no alias tag is found an
     exception is raised.
     """
     if not self._alias:
         alias = self.tags.get('alias')
         if not alias:
             user_data = self._get_user_data(tries=5)
             aliases = user_data.split('|')
             index = self.ami_launch_index
             try:
                 alias = aliases[index]
             except IndexError:
                 log.debug(
                     "invalid user_data: %s (index: %d)" % (aliases, index))
                 alias = None
             if not alias:
                 raise exception.BaseException(
                     "instance %s has no alias" % self.id)
             self.add_tag('alias', alias)
         name = self.tags.get('Name')
         if not name:
             self.add_tag('Name', alias)
         self._alias = alias
     return self._alias

 def connect(self, host=None, username=None, password=None,
             private_key=None, private_key_pass=None, port=None, timeout=30,
             compress=None):
     host = host or self._host
     username = username or self._username
     password = password or self._password
     compress = compress or self._compress
     port = port if port is not None else self._port
     pkey = self._pkey
     if private_key:
         pkey = self.load_private_key(private_key, private_key_pass)
     log.debug("connecting to host %s on port %d as user %s" % (host, port,
                                                                username))
     try:
         sock = self._get_socket(host, port)
         transport = paramiko.Transport(sock)
         transport.banner_timeout = timeout
     except socket.error:
         raise exception.SSHConnectionError(host, port)
     # Enable/disable compression
     transport.use_compression(compress)
     # Authenticate the transport.
     try:
         transport.connect(username=username, pkey=pkey, password=password)
     except paramiko.AuthenticationException:
         raise exception.SSHAuthException(username, host)
     except paramiko.SSHException, e:
         msg = e.args[0]
         raise exception.SSHError(msg)

 def sftp(self):
     """Establish the SFTP connection."""
     if not self._sftp or self._sftp.sock.closed:
         log.debug("creating sftp connection")
         self._sftp = paramiko.SFTPClient.from_transport(self.transport)
         self._sftp.get_channel().settimeout(self._timeout)
     return self._sftp

    def export_fs_to_nodes(self, nodes, export_paths):
        """
        Export each path in export_paths to each node in nodes via NFS

        nodes - list of nodes to export each path to
        export_paths - list of paths on this remote host to export to each node

        Example:
        # export /home and /opt/sge6 to each node in nodes
        $ node.start_nfs_server()
        $ node.export_fs_to_nodes(nodes=[node1,node2],
                                  export_paths=['/home', '/opt/sge6'])
        """
        log.debug("Cleaning up potentially stale NFS entries")
        self.stop_exporting_fs_to_nodes(nodes, paths=export_paths)
        log.info("Configuring NFS exports path(s):\n%s" %
                 ' '.join(export_paths))
        nfs_export_settings = "(async,no_root_squash,no_subtree_check,rw)"
        etc_exports = self.ssh.remote_file('/etc/exports', 'r')
        contents = etc_exports.read()
        etc_exports.close()
        etc_exports = self.ssh.remote_file('/etc/exports', 'a')
        for node in nodes:
            for path in export_paths:
                export_line = ' '.join(
                    [path, node.alias + nfs_export_settings + '\n'])
                if export_line not in contents:
                    etc_exports.write(export_line)
        etc_exports.close()
        self.ssh.execute('exportfs -fra')

 def alias(self):
     """
     Fetches the node's alias stored in a tag from either the instance
     or the instance's parent spot request. If no alias tag is found an
     exception is raised.
     """
     if not self._alias:
         alias = self.tags.get('alias')
         if not alias:
             aliasestxt = self.user_data.get(static.UD_ALIASES_FNAME)
             aliases = aliasestxt.splitlines()[2:]
             index = self.ami_launch_index
             try:
                 alias = aliases[index]
             except IndexError:
                 alias = None
                 log.debug("invalid aliases file in user_data:\n%s" %
                           aliasestxt)
             if not alias:
                 raise exception.BaseException(
                     "instance %s has no alias" % self.id)
             self.add_tag('alias', alias)
         if not self.tags.get('Name'):
             self.add_tag('Name', alias)
         self._alias = alias
     return self._alias

    def get_stats(self):
        """
        this function will ssh to the SGE master and get load & queue stats.
        it will feed these stats to SGEStats, which parses the XML.
        it will return two arrays: one of hosts, each host has a hash with its
        host information inside. The job array contains a hash for every job,
        containing statistics about the job name, priority, etc
        """
        log.debug("starting get_stats")
        master = self._cluster.master_node
        self.stat = SGEStats()

        qhostXml = ""
        qstatXml = ""
        qacct = ""
        try:
            now = self.get_remote_time()
            qatime = self.get_qatime(now)
            qacct_cmd = 'source /etc/profile && qacct -j -b ' + qatime
            qstat_cmd = 'source /etc/profile && qstat -q all.q -u \"*\" -xml'
            qhostXml = '\n'.join(master.ssh.execute( \
                'source /etc/profile && qhost -xml', log_output=False))
            qstatXml = '\n'.join(master.ssh.execute(qstat_cmd,
                                                    log_output=False))
            qacct = '\n'.join(master.ssh.execute(qacct_cmd, log_output=False, \
                                                 ignore_exit_status=True))
        except Exception, e:
            log.error("Error occured getting SGE stats via ssh. "\
                      "Cluster terminated?")
            log.error(e)
            return -1

 def _find_node_for_removal(self):
     """
     This function will find a suitable node to remove from the cluster.
     The criteria for removal are:
     1. The node must not be running any SGE job
     2. The node must have been up for 50-60 minutes past its start time
     3. The node must not be the master, or allow_master_kill=True
     """
     nodes = self._cluster.running_nodes
     to_rem = []
     for node in nodes:
         if not self.allow_master_kill and \
                 node.id == self._cluster.master_node.id:
             log.debug("not removing master node")
             continue
         is_working = self.stat.is_node_working(node)
         mins_up = self._minutes_uptime(node) % 60
         if not is_working:
             log.info("Idle Node %s (%s) has been up for %d minutes " \
                      "past the hour."
                   % (node.id, node.alias, mins_up))
         if self.polling_interval > 300:
             self.kill_after = \
             max(45, 60 - (2 * self.polling_interval / 60))
         if not is_working and mins_up >= self.kill_after:
             to_rem.append(node)
     return to_rem

 def execute(self, args):
     if len(args) < 3:
         self.parser.error("please specify a cluster, remote file or " +
                           "directory, and a local destination path")
     ctag = args[0]
     lpath = args[-1]
     rpaths = args[1:-1]
     cl = self.cm.get_cluster(ctag, load_receipt=False)
     try:
         node = cl.get_node(self.opts.node)
     except exception.InstanceDoesNotExist as ide:
         if self.opts.node == "master":
             #may have happened because master node is clustername-master
             #i.e. dns_prefix = True in config
             #lets check
             try:
                 node = cl.get_node('%s-%s' % (ctag, self.opts.node) )
             except exception.InstanceDoesNotExist as ide2:
                 #k, master is just not there, raise original error
                 log.debug("Neither master nor %s-%s exist." % (ctag, 
                     self.opts.node))
                 raise( ide )
         else:
             #node name was provided
             raise
     if self.opts.user:
         node.ssh.switch_user(self.opts.user)
     for rpath in rpaths:
         if not glob.has_magic(rpath) and not node.ssh.path_exists(rpath):
             raise exception.BaseException(
                 "Remote file or directory does not exist: %s" % rpath)
     node.ssh.get(rpaths, lpath)

    def editStartupScript(self, file, master):
        """
            Add entry in /etc/rc.local to run masterRestart.pl on boot
        """
        log.info("Adding entry to /etc/rc.local to run masterRestart on boot")
        log.debug("startup.StartUp.editStartupScript    self.installdir: %s ", self.installdir)
        if ( file == None or file == "" ):
            file = "/etc/rc.local"
        log.debug("startup.StartUp.editStartupScript    file: %s ", file)

        #### SET RUN resetMaster.pl COMMAND
        command = self.resetdir + "/resetMaster.pl " \
            + " --cell " + self.cell \
            + " --headnodeid " + self.headnodeid \
            + " --cgiscript " + "/cgi-bin/agua/reset.cgi"
        log.debug("startup.StartUp.editStartupScript    command: %s ", command)
        
        #### PRINT COMMAND TO FILE
        infilehandle = master.ssh.remote_file(file, 'r')
        contents = infilehandle.read()
        log.debug("startup.StartUp.editStartupScript    contents: %s ", contents)
        contents = string.replace(contents, "exit 0", "")
        contents = string.replace(contents, command, "")
        contents += command + "\n"
        contents += "\nexit 0\n"
        log.debug("startup.StartUp.editStartupScript    printing to %s contents: %s ", file, contents)

        outfilehandle = master.ssh.remote_file(file, 'w')
        outfilehandle.write(contents)
        outfilehandle.close()

 def __init__(self, my_arg, my_other_arg, my_other_other_arg):
     self.my_arg = my_arg
     self.my_other_arg = my_other_arg
     self.my_other_other_arg = my_other_other_arg
     msg = "setupclass3: my_arg = %s, my_other_arg = %s"
     msg += " my_other_other_arg = %s"
     log.debug(msg % (my_arg, my_other_arg, my_other_other_arg))

 def __init__(self,enable_hvmem="True",master_slots=0):
     if enable_hvmem == "False":
         self.enable_hvmem = False
     else:
         self.enable_hvmem = True
     self.master_slots = master_slots
     log.debug("enable_hvmem = %s , master_slots = %s" % (self.enable_hvmem, self.master_slots))

    def mount_nfs_shares(self, server_node, remote_paths):
        """
        Mount each path in remote_paths from the remote server_node

        server_node - remote server node that is sharing the remote_paths
        remote_paths - list of remote paths to mount from server_node
        """
        self.ssh.execute('/etc/init.d/portmap start')
        # TODO: move this fix for xterm somewhere else
        self.ssh.execute('mount -t devpts none /dev/pts',
                         ignore_exit_status=True)
        mount_map = self.get_mount_map()
        mount_paths = []
        for path in remote_paths:
            network_device = "%s:%s" % (server_node.alias, path)
            if network_device in mount_map:
                mount_path, typ, options = mount_map.get(network_device)
                log.debug('nfs share %s already mounted to %s on '
                          'node %s, skipping...' %
                          (network_device, mount_path, self.alias))
            else:
                mount_paths.append(path)
        remote_paths = mount_paths
        remote_paths_regex = '|'.join(map(lambda x: x.center(len(x) + 2),
                                          remote_paths))
        self.ssh.remove_lines_from_file('/etc/fstab', remote_paths_regex)
        fstab = self.ssh.remote_file('/etc/fstab', 'a')
        for path in remote_paths:
            fstab.write('%s:%s %s nfs vers=3,user,rw,exec,noauto 0 0\n' %
                        (server_node.alias, path, path))
        fstab.close()
        for path in remote_paths:
            if not self.ssh.path_exists(path):
                self.ssh.makedirs(path)
            self.ssh.execute('mount %s' % path)

 def get(self, remotepaths, localpath=''):
     """
     Copies one or more files from the remote host to the local host.
     """
     remotepaths = self._make_list(remotepaths)
     localpath = localpath or os.getcwd()
     globs = []
     noglobs = []
     for rpath in remotepaths:
         if glob.has_magic(rpath):
             globs.append(rpath)
         else:
             noglobs.append(rpath)
     globresults = [self.glob(g) for g in globs]
     remotepaths = noglobs
     for globresult in globresults:
         remotepaths.extend(globresult)
     recursive = False
     for rpath in remotepaths:
         if not self.path_exists(rpath):
             raise exception.BaseException(
                 "Remote file or directory does not exist: %s" % rpath)
     for rpath in remotepaths:
         if self.isdir(rpath):
             recursive = True
             break
     try:
         self.scp.get(remotepaths, local_path=localpath,
                      recursive=recursive)
     except Exception, e:
         log.debug("get failed: remotepaths=%s, localpath=%s",
                   str(remotepaths), localpath)
         raise exception.SCPException(str(e))

    def run(self, nodes, master, user, user_shell, volumes):
        """
            Mount NFS shares on master and all nodes
        """
        log.info("Running plugin automount")
        log.debug("automount.NfsShares.run    automount.NfsShares.run(nodes, master, user, user_shell, volumes)")

        #### OPEN NFS-RELATED PORTS FOR THIS CLUSTER
        self.openNfsPorts("default")
        self.openNfsPorts('@sc-' + self.cluster)

        #### SET HEAD NODE INTERNAL IP
        self.getHeadIp();

        #### FIX mountd PORT ON head AND MASTER/NODES
        mountdport = "32767"
        for node in nodes:
            self.setMountdOnNode(node, mountdport)
        
        self.setMountdOnHead(mountdport)
        self.restartServicesOnHead()

        #### MOUNT ON ALL NODES
        for node in nodes:
            self.mount(node)

        log.info("Completed plugin automount")

 def scp(self):
     """Initialize the SCP client."""
     if not self._scp or not self._scp.transport.is_active():
         log.debug("creating scp connection")
         self._scp = scp.SCPClient(self.transport,
                                   progress=self._file_transfer_progress)
     return self._scp

 def setMountdOnNode(self, node, mountdport):
     """
         Fix mountd port to same number on all hosts - head, master and exec nodes
     """
     log.info("Setting mountd port on %s", node.alias)
     cmd = self.mountdCommand(mountdport)
     log.debug("Doing node.ssh.execute: " + cmd)
     node.ssh.execute(cmd)