def stop(self):
     print('Stopping service: ', self.name)
     if settings.TEST_SERVER == 'localhost':
         pids = sh.pgrep('-f', 'manage.py runserver', _ok_code=[0, 1])
         for pid in pids:
             sh.kill(pid.rstrip())
     else:
         sh.ssh(settings.TEST_SERVER, 'sudo supervisorctl stop all')

def ssh_interact(char, aggregated):
    """automate connecting to an ssh server"""
    stdout.write(char.encode())
    aggregated += char
    if aggregated.endswith("password: "):
        stdin.write("correcthorsebatterystaple\n")
        _out = os.fdopen(sys.stderr, "wb", 0)
        ssh("9.10.10.100", _out=ssh_interact, _out_bufsize=0, _tty_in=True)
    return _out

    def start(self):
        print('Starting service: ', self.name)
        if settings.TEST_SERVER == 'localhost':
            self._run_manage_py('runserver', bg=True)
        else:
            sh.ssh(settings.TEST_SERVER, 'sudo supervisorctl start all')

        for i in range(5):
            time.sleep(1)
            if self.is_running():
                return

        raise Exception("Service not running after 5 seconds")

def ssh_command(key, dns, command):
    try:
        res = ssh("-oConnectTimeout=15", "-i", key, cluster_config.USER + "@" + dns, command)
        return res
    except Exception as e:
        print "Command failed", e
        return 

 def _qmgr(self, command):
     result = None
     try:
         result = ssh("{0}@{1}".format(self.user, self.host), command)
     except:
         raise RuntimeError("can not execute qmgr via ssh {0}".format(command))
     return result

    def qstat(self, refresh=True):
        if self.pbs_qstat_data is None or refresh:
            try:
                xmldata = str(ssh("{0}@{1}".format(self.user, self.host), "qstat", "-x"))
            except:
                raise RuntimeError("can not execute pbs qstat via ssh")
            info = {}

            try:
                xmldoc = minidom.parseString(xmldata)

                itemlist = xmldoc.getElementsByTagName('Job')
                for item in itemlist:
                    job = {}
                    for attribute in item.childNodes:
                        if len(attribute.childNodes) == 1:
                            job[attribute.nodeName] = attribute.firstChild.nodeValue
                        else:
                            job[attribute.nodeName] = {}
                            for subchild in attribute.childNodes:
                                job[attribute.nodeName][subchild.nodeName] = subchild.firstChild.nodeValue

                    info[job['Job_Id']] = job
            except:
                pass
            self.pbs_qstat_data = info
        #return self.pbs_qstat_data

        #pprint (self.pbs_qstat_data)

        if self.cluster_queues is None:
            return {self.host: self.pbs_qstat_data}
        else:
            return self.qstat_extract(self.cluster_queues, self.pbs_qstat_data)

 def delete_user_repo(self):
     """delete user's remote repository"""
     conf = self.configuration
     dst_repo_name = conf.get('buildbot-configs', 'dst_repo_name')
     if not dst_repo_name.endswith(self.bug):
         msg = "cowardly refusing to delete {0}".format(dst_repo_name)
         msg = "{0}, its name does not end with {1}".format(msg, self.bug)
         log.error(msg)
         raise BuildbotConfigsError(msg)
     cmd = ("hg.mozilla.org", "edit", dst_repo_name,  "delete", "YES")
     log.info('deleting {0}'.format(dst_repo_name))
     log.debug('running ssh {0}'.format(' '.join(cmd)))
     output = []
     try:
         for line in sh.ssh(cmd, _iter=True):
             out = line.strip()
             log.debug(out)
             output.append(out)
     except sh.ErrorReturnCode_1:
         log.debug('trying to delete a non existing repo... pass')
         pass
     except sh.ErrorReturnCode:
         msg = 'bad exit code executing {0}'.format(' '.join(cmd))
         log.error(msg)
         raise BuildbotConfigsError(msg)

def task_sensors(dict_idip):
    dict_data = {}
    for cluster in dict_idip.keys():
        dict_data[cluster] = {}
        for uid in dict_idip[cluster].keys():
            ip = cluster[uid]
            dict_data[cluster][uid]["ip"] = ip
            # fetch uid-ip server's temperature
            report = ssh("-o ConnectTimeout=1", "-o ConnectionAttempts=1", "user", ip, "sensors")
            temp = parseCpuTemperature(report)
            dict_data[cluster][uid]["temp"] = temp[0]

    # write current temperature data to mongo db

    # get the highest cpu temperature throught parseing the output of 'sensors'
    # return is a list including 2 elems, [36.0, C] or [36.0, F]
    # C or F is the unit name of temperature
    def parseCpuTemperature(self, values):
        lines = values.split("\n")
        cpu_lines = [x for x in lines if x.find("(high") > -1]
        tunit = "C"
        tmax = -1
        for line in cpu_lines:
            # position of degree sign
            pos_degree = line.find(u"\xb0")
            # position of +
            pos_plus = line.find(u"+")
            tnum = float(line[pos_plus + 1 : pos_degree])
            tunit = line[pos_degree + 1 : pos_degree + 2]
            if tnum > tmax:
                tmax = tnum

        return [tmax, tunit]

    def __call__(self, *args, **kwargs):
        self.process = ssh('{}@{}'.format(self.user, self.host), '-p', self.port,
                            *args,
                            _out=self.out_iteract, _out_bufsize=0, _tty_in=True,
                            _err=self.err_iteract, **kwargs)

        super().__call__(*args, **kwargs)

 def execute(self, name, command):
     """executes the command on the named host"""
     if name in ["localhost"]:
         r = '\n'.join(sh.sh("-c", command).split()[-1:])
     else:
         r = '\n'.join(sh.ssh(name, command).split()[-1:])
     return r

 def _cmd(self, *args):
     cmd = []
     cmd.extend(['-p', self.gerrit_port,
                 "{0}@{1}".format(self.gerrit_user, self.gerrit_host),
                 'gerrit'])
     cmd.extend(args)
     log.debug(cmd)
     return ssh(*cmd)

 def _run_manage_py(self, command, *args, **kwargs):
     python = '{}/bin/python'.format(self.settings['PYTHON_ENV'])
     manage = '{}/manage.py'.format(self.settings['ENVIRONMENT_ROOT'])
     try:
         if settings.TEST_SERVER == 'localhost':
             with cd(self.settings['ENVIRONMENT_ROOT']):
                 sh.Command(python)(manage, command, _bg=kwargs.get('bg', False), *args)
         else:
             sh.ssh(settings.TEST_SERVER, '{ssh_command} {manage} {manage_command} {args}'.format(
                 ssh_command='cd {} && {}'.format(self.settings['ENVIRONMENT_ROOT'], python),
                 manage=manage,
                 manage_command=command,
                 args=' '.join(args)), _iter=True)
     except Exception as e:
         if hasattr(e, 'stderr'):
             print(e.stderr)
         raise

	def get_status(self, jobid):
		""".. function:: get_status(jobid)

		      Return the current status of the job referenced by the given jobid

		      :param jobid: id of job to check on host"""

		result = ssh(self.host, "checkjob", jobid)
		return result

    def qinfo(self, refresh=True):
        '''
        returns qstat -Q -f in dict format
        
        :param refresh: refreshes the qinfo
        :type refresh: Boolean
        '''
        
        if self.pbs_qinfo_data is None or refresh:
            try:
                result = ssh("{0}@{1}".format(self.user, self.host), "qstat -Q -f")
            except:
                raise RuntimeError("can not execute pbs qstat via ssh")

            d = {}

            # sanitize block

            result = result.replace("\n\t", "")

            result = result.replace('resources_assigned.', 'resources_assigned_')
            result = result.replace('resources_default.', 'resources_default_')
            result = result.replace('resources_max.', 'resources_max_')


            for block in result.split("\n\n")[:-1]:
                block = [x.replace(" =", ":", 1) for x in block.split("\n")]
                block[0] = block[0].replace("Queue: ", "") + ":"
                queue = block[0][:-1]

                block = '\n'.join(block)

                block_yaml = yaml.safe_load(block)
                d[queue] = block_yaml[queue]

                d[queue]['queue'] = queue
                # end sanitize

                if 'state_count' in d[queue]:
                    values = [x.split(":") for x in d[queue]['state_count'].split(" ")]
                    d[queue]['state_count'] = {}
                    for value in values:
                        d[queue]['state_count'][value[0]] = value[1]

                if 'acl_hosts' in d[queue]:
                    # print d[queue]['acl_hosts']
                    d[queue]['acl_hosts'] = d[queue]['acl_hosts'].split("+")

        self.pbs_qinfo_data = d

        #pprint(self.qinfo_extract(self.cluster_queues, self.pbs_qinfo_data))
        
        if self.cluster_queues is None:
            return {self.host: self.pbs_qinfo_data}
        else:
            return self.qinfo_extract(self.cluster_queues, self.pbs_qinfo_data)

    def open_port_forwarding(self):
        """
        Opens correct ports in remote for forwarding.
        :return:
        """

        base_command = "ssh %(user)[email protected]%(server)s -f -N -L %(port)s:%(ip)s:%(port)s"

        # open stdin_port
        sh.ssh(base_command
               % {"user": self.user,
                  "server": self.server,
                  "port": self.kernel_data.get("stdin_port"),
                  "ip": self.kernel_data.get("ip")})

        # open control_port
        sh.ssh(base_command
               % {"user": self.user,
                  "server": self.server,
                  "port": self.kernel_data.get("control_port"),
                  "ip": self.kernel_data.get("ip")})

        # open hb_port
        sh.ssh(base_command
               % {"user": self.user,
                  "server": self.server,
                  "port": self.kernel_data.get("hb_port"),
                  "ip": self.kernel_data.get("ip")})

        # open shell_port
        sh.ssh(base_command
               % {"user": self.user,
                  "server": self.server,
                  "port": self.kernel_data.get("shell_port"),
                  "ip": self.kernel_data.get("ip")})

        # open iopub_port
        sh.ssh(base_command
               % {"user": self.user,
                  "server": self.server,
                  "port": self.kernel_data.get("iopub_port"),
                  "ip": self.kernel_data.get("ip")})

    def create_repo(self):
        """creates a buildbot-configs repo as
           a copy of https://hg.mozilla.org/build/buildbot-configs/
           It creates hg.m.o/users/<ldap>_mozilla.com/buildbot-configs-<bug>
           appending -<bug> because this script creates and destroys that
           repo without asking for user permission.

        """
        conf = self.configuration
        src_repo_name = conf.get('buildbot-configs', 'src_repo_name')
        dst_repo_name = conf.get('buildbot-configs', 'dst_repo_name')
        if not dst_repo_name.endswith(self.bug):
            msg = "cowardly refusing to clone {0}".format(dst_repo_name)
            msg = "{0}, its name does not end with {1}".format(msg, self.bug)
            log.error(msg)
            raise BuildbotConfigsError(msg)
        cmd = ("hg.mozilla.org", "clone", dst_repo_name,  src_repo_name)
        log.info('cloning {0} to {1}'.format(src_repo_name, dst_repo_name))
        log.debug('running ssh {0}'.format(' '.join(cmd)))
        sh.ssh(cmd)

def ssh_tunnel(nodes, passwd=None):
	""" Sets up an SSH tunnel to a given set of nodes

	Args:
		nodes (list): List of node IDs on which the image should be installed

	Note: It will prompt for ssh password.

		DO NOT EXPECT THIS FUNCTION TO EXIT.
	"""
	if passwd is not None:
		global __ssh_pass__
		__ssh_pass__ = passwd
	args = ["-nNxT4"]
	for node in nodes:
		args.append("-L")
		args.append("9{id:03d}:localhost:9{id:03d}".format(id=node))
	args.append("[email protected]")
	ssh(args, _out=__ssh_interact__, _out_bufsize=0, _tty_in=True)
	ssh.wait()

    def __call__(self, *args, **kwargs):
        self.process = ssh(
                            '-o UserKnownHostsFile=/dev/null',
                            '-o StrictHostKeyChecking=no',
                            '-o LogLevel=quiet',
                            '{0}@{1}'.format(self.user, self.host),
                            '-p', self.port,
                            'LANG=C', *args,
                            _out=self.out_iteract, _out_bufsize=0, _tty_in=True,
                            **kwargs)

        super(SSHController, self).__call__(*args, **kwargs)

    def get_ipmi_temperature(self, hostname):

        (hostaddr, data) = self.inventory.ipadr_cluster(hostname, "bmc")
        clustername = data["cm_cluster"]
        config = self.config_server[clustername]['bmc']
        # username and password to access ipmitool
        password = config['password']
        username = config['user']

        command = "ipmitool -I lanplus -H {0} -U {1} -P {2} sdr type temperature".format(
            hostaddr, username, password)
        # access ipmitool need a proxy server
        proxyaddr = config['proxy']['ip']
        proxyusername = config['proxy']['user']

        log.debug("Get temperature for host '{2}' via proxy server '{0}@{1}'".format(
            proxyusername, proxyaddr, hostname))

        try:
            result = ssh("{0}@{1}".format(proxyusername, proxyaddr), command)
        except:
            result = ""

        dict_result = None
        if result == "":
            log.warning(
                "Cannot access to host '{0}' OR ipmitool failed on host '{0}'".format(hostname))
        else:
            log.debug(
                "Temperature data retrieved from host '{0}' successfully.".format(hostname))
            dict_result = {}
            lines = result.split("\n")
            for line in lines:
                fields = map(lambda x: x.strip(), line.split("|"))
                name = fields[0]
                # test and ignore the blank line in the last output
                if name == "":
                    continue
                value = "-1"
                unit = "C"
                m = self.patt.match(fields[4])
                if m:
                    value = m.group(1)
                    unit = m.group(2)
                dict_result[name] = {"name": fields[0],
                                     "address": fields[1],
                                     "status": fields[2],
                                     "entity": fields[3],
                                     "value": value,
                                     "unit": unit
                                     }
        return dict_result

    def create_repo(self):
        """creates a reposiory as
           a copy of https://hg.mozilla.org/build/self.name/
           It creates hg.m.o/users/<ldap>_mozilla.com/self.name-<bug>
           appending -<bug> because this script creates and destroys that
           repo without asking for user permission.

        """
        conf = self.configuration
        src_repo_name = conf.get(self.name, 'src_repo_name')
        dst_repo_name = conf.get(self.name, 'dst_repo_name')
#        if not dst_repo_name.endswith(self.bug):
#            msg = "cowardly refusing to clone {0}".format(dst_repo_name)
#            msg = "{0}, its name does not end with {1}".format(msg, self.bug)
#            log.error(msg)
#            raise RepositoryError(msg)
        # added more robust check on pushing rather than cloning...

        cmd = ('hg.mozilla.org', 'clone', dst_repo_name,  src_repo_name)
        log.info('cloning {0} to {1}'.format(src_repo_name, dst_repo_name))
        log.debug('running ssh {0}'.format(' '.join(cmd)))
        ssh(cmd)