def request(self, method, url, body, headers, content_type=None):
        """Low-level networking. All HTTP-method methods call this"""
        if headers is None:
            headers = {}

        if content_type is not None:
            headers['Content-Type'] = content_type

        if self.auth_header is not None:
            headers['Authorization'] = self.auth_header
        headers['User-Agent'] = self.user_agent
        fancylogger.getLogger().debug('cli request: %s, %s, %s, %s', method, url, body, headers)
        # TODO: in recent python: Context manager
        conn = self.get_connection(method, url, body, headers)
        status = conn.code
        if method == self.HEAD:
            pybody = conn.headers
        else:
            body = conn.read()
            try:
                pybody = json.loads(body)
            except ValueError:
                pybody = body
        fancylogger.getLogger().debug('reponse len: %s ', len(pybody))
        conn.close()
        return status, pybody

    def test_get_callers_logger(self):
        """Test get_callers_logger function."""
        # returns None if no logger is available
        self.assertEqual(get_callers_logger(), None)

        # find defined logger in caller's context
        logger = getLogger("foo")
        callers_logger = get_callers_logger()
        # result depends on whether tests were run under 'python' or 'python -O'
        self.assertTrue(callers_logger in [logger, None])

        # also works when logger is 'higher up'
        class Test(object):
            """Dummy test class"""

            def foo(self, logger=None):
                """Dummy test method, returns logger from calling context."""
                return get_callers_logger()

        test = Test()
        self.assertTrue(logger, [test.foo(), None])

        # closest logger to caller is preferred
        logger2 = getLogger(test.__class__.__name__)
        self.assertTrue(logger2 in [test.foo(logger=logger2), None])

    def test_parentinfo(self):
        """Test the collection of parentinfo"""
        log_fr = fancylogger.getLogger(fname=False)  # rootfancylogger
        pi_fr = log_fr._get_parent_info()
        self.assertEqual(len(pi_fr), 2)

        log_l1 = fancylogger.getLogger('level1', fname=False)
        # fname=False is required to have the naming similar for child relations
        pi_l1 = log_l1._get_parent_info()
        self.assertEqual(len(pi_l1), 3)

        py_v_27 = sys.version_info >= (2, 7, 0)
        if py_v_27:
            log_l2a = log_l1.getChild('level2a')
            pi_l2a = log_l2a._get_parent_info()
            self.assertEqual(len(pi_l2a), 4)

        # this should be identical to getChild
        log_l2b = fancylogger.getLogger('level1.level2b', fname=False)
        # fname=False is required to have the name similar
        # cutoff last letter (a vs b)
        if py_v_27:
            self.assertEqual(log_l2a.name[:-1], log_l2b.name[:-1])
        pi_l2b = log_l2b._get_parent_info()
        # yes, this broken on several levels (incl in logging itself)
        # adding '.' in the name does not automatically create the parent/child relations
        # if the parent with the name exists, this works
        self.assertEqual(len(pi_l2b), 4)

        log_l2c = fancylogger.getLogger('level1a.level2c', fname=False)
        pi_l2c = log_l2c._get_parent_info()
        self.assertEqual(len(pi_l2c), 3)  # level1a as parent does not exist

    def test_raiseException(self):
        """Test raiseException log method."""
        # truncate the logfile
        open(self.logfn, 'w')

        def test123(exception, msg):
            """Utility function for testing raiseException."""
            try:
                raise exception(msg)
            except:
                logger.raiseException('HIT')

        logger = fancylogger.getLogger('fail_test')
        self.assertErrorRegex(Exception, 'failtest', test123, Exception, 'failtest')
        self.assertTrue(re.match("^WARNING.*HIT.*failtest\n.*in test123.*$", open(self.logfn, 'r').read(), re.M))

        open(self.logfn, 'w')
        fancylogger.FancyLogger.RAISE_EXCEPTION_CLASS = KeyError
        logger = fancylogger.getLogger('fail_test')
        self.assertErrorRegex(KeyError, 'failkeytest', test123, KeyError, 'failkeytest')
        self.assertTrue(re.match("^WARNING.*HIT.*'failkeytest'\n.*in test123.*$", open(self.logfn, 'r').read(), re.M))

        open(self.logfn, 'w')
        fancylogger.FancyLogger.RAISE_EXCEPTION_LOG_METHOD = lambda c, msg: c.warning(msg)
        logger = fancylogger.getLogger('fail_test')
        self.assertErrorRegex(AttributeError, 'attrtest', test123, AttributeError, 'attrtest')
        self.assertTrue(re.match("^WARNING.*HIT.*attrtest\n.*in test123.*$", open(self.logfn, 'r').read(), re.M))

    def test_fancylogger_as_rootlogger_logging(self):
        """
        Test if just using import logging, logging with logging uses fancylogger
        after setting the root logger
        """

        # test logging.root is loggin root logger
        # this is an assumption made to make the fancyrootlogger code work
        orig_root = logging.getLogger()
        self.assertEqual(logging.root, orig_root,
                         msg='logging.root is the root logger')
        self.assertFalse(isinstance(logging.root, fancylogger.FancyLogger),
                         msg='logging.root is not a FancyLogger')


        stringfile = StringIO()
        sys.stderr = stringfile
        handler = fancylogger.logToScreen()
        fancylogger.setLogLevelDebug()
        logger = fancylogger.getLogger()

        self.assertEqual(logger.handlers, [self.handler, handler],
                         msg='active handler for root fancylogger')
        self.assertEqual(logger.level, fancylogger.getLevelInt('DEBUG'), msg='debug level set')

        msg = 'this is my string'
        logging.debug(msg)
        self.assertEqual(stringfile.getvalue(), '',
                         msg="logging.debug reports nothing when fancylogger loglevel is debug")

        fancylogger.setroot()
        self.assertTrue(isinstance(logging.root, fancylogger.FancyLogger),
                         msg='logging.root is a FancyLogger after setRootLogger')
        self.assertEqual(logging.root.level, fancylogger.getLevelInt('DEBUG'), msg='debug level set for root')
        self.assertEqual(logger.level, logging.NOTSET, msg='original root fancylogger level set to NOTSET')

        self.assertEqual(logging.root.handlers, [self.handler, handler],
                         msg='active handler for root logger from previous root fancylogger')
        self.assertEqual(logger.handlers, [], msg='no active handlers on previous root fancylogger')

        root_logger = logging.getLogger('')
        self.assertEqual(root_logger, logging.root,
                        msg='logging.getLogger() returns logging.root FancyLogger')

        frl = fancylogger.getLogger()
        self.assertEqual(frl, logging.root,
                        msg='fancylogger.getLogger() returns logging.root FancyLogger')

        logging.debug(msg)
        self.assertTrue(msg in stringfile.getvalue(),
                         msg="logging.debug reports when fancylogger loglevel is debug")

        fancylogger.resetroot()
        self.assertEqual(logging.root, orig_root,
                         msg='logging.root is the original root logger after resetroot')

        # restore
        fancylogger.logToScreen(enable=False, handler=handler)

 def getCluster(name):
     """
     static factory method, should be in Cluster class
     returns a cluster object of the given name
     """
     for cls in Cluster.__subclasses__():
         if cls._is_cluster_for(name):
             return cls()
     fancylogger.getLogger("clusters.getCluster").raiseException("No such cluster %s" % name, NoSuchClusterException)

    def get_job(classname, options):
        """
        This is a job factory.

        Returns an instance of classname initialized with options
        """
        for cls in get_subclasses(Job):
            if cls._is_job_for(classname):
                return cls(options)
        getLogger().error("No job class found for %s", classname)

def main():
    """Main function"""
    try:
        instance_options = get_mpi_and_sched_and_options()
        if instance_options:
            instance = getinstance(*instance_options)
            instance.main()
    except Exception:
        fancylogger.getLogger().exception("Main failed")
        sys.exit(1)

 def get_connection(self, method, url, body, headers):
     if not self.url.endswith('/') and not url.startswith('/'):
         sep = '/'
     else:
         sep = ''
     request = urllib2.Request(self.url + sep + url, data=body)
     for header, value in headers.iteritems():
         request.add_header(header, value)
     request.get_method = lambda: method
     fancylogger.getLogger().debug('opening request:  %s%s%s', self.url, sep, url)
     connection = self.opener.open(request)
     return connection

    def test_error_env_options(self):
        """Test log error on unknown environment option"""
        self.reset_logcache()
        mylogger = fancylogger.getLogger('ExtOptionParser')
        mylogger.error = self.mock_logmethod(mylogger.error)
        mylogger.debug = self.mock_logmethod(mylogger.debug)

        self.assertEqual(self.count_logcache('error'), 0)

        os.environ['GENERALOPTIONTEST_XYZ'] = '1'
        topt1 = TestOption1(go_args=['--level-level'], envvar_prefix='GENERALOPTIONTEST')
        self.assertEqual(self.count_logcache('error'), 0,
                         msg='no errors logged, got %s' % self.count_logcache('error'))

        topt1 = TestOption1(go_args=['--level-level'], envvar_prefix='GENERALOPTIONTEST', error_env_options=True)
        print self.LOGCACHE['error']
        self.assertEqual(self.count_logcache('error'), 1,
                         msg='one error should be logged, got %s' % self.count_logcache('error'))

        # using a custom error method
        def raise_error(msg, *args):
            """Raise error with given message and string arguments to format it."""
            raise Exception(msg % args)

        self.assertErrorRegex(Exception, "Found 1 environment variable.* prefixed with GENERALOPTIONTEST", TestOption1,
                              go_args=['--level-level'], envvar_prefix='GENERALOPTIONTEST', error_env_options=True,
                              error_env_option_method=raise_error)

def det_pylibdir(plat_specific=False, python_cmd=None):
    """Determine Python library directory."""
    log = fancylogger.getLogger('det_pylibdir', fname=False)

    if python_cmd is None:
        # use 'python' that is listed first in $PATH if none was specified
        python_cmd = 'python'

    # determine Python lib dir via distutils
    # use run_cmd, we can to talk to the active Python, not the system Python running EasyBuild
    prefix = '/tmp/'
    args = 'plat_specific=%s, prefix="%s"' % (plat_specific, prefix)
    pycode = "import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(%s))" % args
    cmd = "%s -c '%s'" % (python_cmd, pycode)

    log.debug("Determining Python library directory using command '%s'", cmd)

    out, ec = run_cmd(cmd, simple=False, force_in_dry_run=True)
    txt = out.strip().split('\n')[-1]

    # value obtained should start with specified prefix, otherwise something is very wrong
    if not txt.startswith(prefix):
        raise EasyBuildError("Last line of output of %s does not start with specified prefix %s: %s (exit code %s)",
                             cmd, prefix, out, ec)

    pylibdir = txt[len(prefix):]
    log.debug("Determined pylibdir using '%s': %s", cmd, pylibdir)
    return pylibdir

    def __init__(self, options):
        self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)

        self.vars = {"cwd": None, "jobid": None}
        self.jobid = None

        self.job_filter = None

def main():
    """
    main method
    parses arguments and constructs Manager object and performs actions
    """
    logger = fancylogger.getLogger()
    options = get_options()
    logger.debug("arguments parsed, starting manager")
    manager = Manager(options)

    #print status
    if manager.status:
        print "Status:\n %s" % manager.status

    #actually do requested commands
    out = manager.doit()
    #parse and display the output
    errors = []
    if out:
        for i in out:  # this is an array of nodes and their  output,err
            if len(i[1]) > 0:  # only print if something to show
                print "%s:" % i[0]  # first element is the node
                for j in i[1]:  # second element is an array of [outputs of commands,errors]
                    print "    output: %s" % str(j[1][0])
                    if j[1][1]:
                        print "    error:  %s" % str(j[1][1])
                        errors.append(i[0])

    # print all nodes with errors out on the end
    if len(errors) > 0:
        print "ERRORS occured in: \n ", ",".join([str(x) for x in errors])

 def request(self, method, url, body, headers):
     if self.auth_header is not None:
         headers['Authorization'] = self.auth_header
     headers['User-Agent'] = self.user_agent
     fancylogger.getLogger().debug('cli request: %s, %s, %s, %s', method, url, body, headers)
     #TODO: in recent python: Context manager
     conn = self.get_connection(method, url, body, headers)
     status = conn.code
     body = conn.read()
     try:
         pybody = json.loads(body)
     except ValueError:
         pybody = body
     fancylogger.getLogger().debug('reponse len: %s ', len(pybody))
     conn.close()
     return status, pybody

    def __init__(self, options=None, **kwargs):
        if not hasattr(self, 'log'):
            self.log = getLogger(self.__class__.__name__)
        if not hasattr(self, 'options'):
            self.options = options

        self.cores_per_node = None
        self.set_cores_per_node()

        self.sched_id = None
        self.set_sched_id()

        self.cpus = []
        self.set_cpus()

        self.nodes, self.nodes_uniq, self.nodes_tot_cnt = None, None, None
        self.set_nodes()

        self.multiplier = None
        self.set_multiplier()

        ppn = os.environ.get('PBS_NUM_PPN')
        if ppn is not None:
            self.ppn = int(ppn)
            self.log.debug("Determined # cores per node via $PBS_NUM_PPN: %s" % self.ppn)
        else:
            self.ppn = len(self.cpus)
            self.log.debug("Failed to determine # cores per node via $PBS_NUM_PPN, using affinity: found %s" % self.ppn)
        self.set_ppn()

        self.mpinodes = None
        self.set_mpinodes()

        super(Sched, self).__init__(**kwargs)

    def __init__(self, cmd=None, **kwargs):
        """
        Handle initiliastion
            @param cmd: command to run
            @param input: set "simple" input
            @param startpath: directory to change to before executing command
            @param disable_log: use fake logger (won't log anything)
        """
        self.input = kwargs.pop('input', None)
        self.startpath = kwargs.pop('startpath', None)
        if kwargs.pop('disable_log', None):
            self.log = DummyFunction()  # No logging
        if not hasattr(self, 'log'):
            self.log = getLogger(self._get_log_name())

        self.cmd = cmd  # actual command

        self._cwd_before_startpath = None

        self._process_module = None
        self._process = None

        self.readsize = 1024  # number of bytes to read blocking

        self._shellcmd = None
        self._popen_named_args = None

        self._process_exitcode = None
        self._process_output = None

        self._post_exitcode_log_failure = self.log.error

        super(Run, self).__init__(**kwargs)

    def __init__(self):
        self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)

        self.hadoopversion = {
            'major':-1,
            'minor':-1,
            'small':-1,
            'suffix': None,
        }

        self.hadoophome = None  # hadoop home (could be $EBROOTHADOOP)
        self.hadoop = None

        self.javaversion = {
            'major':-1,
            'minor':-1,
            'suffix': None,
        }
        self.java = None
        self.javahome = None

        self.name = 'all'  # default task start-all.sh, stop-all.sh
        self.daemonname = 'hadoop'

        self.start_script = None
        self.stop_script = None
        self.daemon_script = None

        self.extrasearchpaths = []

    def __init__(self, script, name, env_vars=None, hours=None, cores=None):
        """Create a new Job to be submitted to SLURM."""
        self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)

        self.jobid = None
        self.script = script
        self.name = name

        self.job_specs = {
            'job-name': self.name,
            # pattern for output file for submitted job;
            # SLURM replaces %x with job name, %j with job ID (see https://slurm.schedmd.com/sbatch.html#lbAF)
            'output': '%x-%j.out',
            'wrap': self.script,
        }

        if env_vars:
            self.job_specs['export'] = ','.join(sorted(env_vars.keys()))

        max_walltime = build_option('job_max_walltime')
        if hours is None:
            hours = max_walltime
        if hours > max_walltime:
            self.log.warn("Specified %s hours, but this is impossible. (resetting to %s hours)" % (hours, max_walltime))
            hours = max_walltime
        self.job_specs['time'] = hours * 60

        if cores:
            self.job_specs['nodes'] = 1
            self.job_specs['ntasks'] = cores
            self.job_specs['ntasks-per-node'] = cores
        else:
            self.log.warn("Number of cores to request not specified, falling back to whatever Slurm does by default")

    def test_deprecated(self):
        """Test deprecated log function."""
        # truncate the logfile
        open(self.logfn, 'w')

        # log message
        logger = fancylogger.getLogger('deprecated_test')

        max_ver = "1.0"

        # test whether deprecation works
        msgre_tpl_error = r"DEPRECATED\s*\(since v%s\).*%s" % (max_ver, MSG)
        self.assertErrorRegex(Exception, msgre_tpl_error, logger.deprecated, MSG, "1.1", max_ver)
        self.assertErrorRegex(Exception, msgre_tpl_error, logger.deprecated, MSG, "1.0", max_ver)

        # test whether deprecated warning works
        # no deprecation if current version is lower than max version
        logger.deprecated(MSG, "0.9", max_ver)
        msgre_tpl_warning = r"WARNING.*DEPRECATED\s*\(since v%s\).*%s" % (max_ver, MSG)
        msgre_warning = re.compile(msgre_tpl_warning)
        txt = open(self.logfn, 'r').read()
        self.assertTrue(msgre_warning.search(txt))

        # test handling of non-UTF8 chars
        msg = MSG + u"\x81"
        msgre_tpl_error = r"DEPRECATED\s*\(since v%s\).*\xc2\x81" % max_ver
        self.assertErrorRegex(Exception, msgre_tpl_error, logger.deprecated, msg, "1.1", max_ver)
        logger.deprecated(msg, "0.9", max_ver)
        txt = open(self.logfn, 'r').read()
        self.assertTrue(msgre_warning.search(txt))

    def test_uft8_decoding(self):
        """Test UTF8 decoding."""
        # truncate the logfile
        open(self.logfn, 'w')

        logger = fancylogger.getLogger('utf8_test')
        logger.setLevel('DEBUG')

        msgs = [
            # bytestrings
            "This is a pure ASCII text.",  # pure ASCII
            "Here are some UTF-8 characters: ß, ©, Ω, £.",  # only UTF8 characters
            "This non-UTF-8 character '\x80' should be handled properly.",  # contains non UTF-8 character
            # unicode strings
            u"This is a pure ASCII text.",  # pure ASCII
            u"Here are some UTF8 characters: ß, ©, Ω, £.",  # only UTF8 characters
            u"This non-UTF8 character '\x80' should be handled properly.",  # contains non UTF-8 character
        ]
        for msg in msgs:
            logger.critical(msg)
            logger.debug(msg)
            logger.error(msg)
            logger.exception(msg)
            logger.fatal(msg)
            logger.info(msg)
            logger.warning(msg)
            logger.warn(msg)
            if isinstance(msg, unicode):
                regex = msg.encode('utf8', 'replace')
            else:
                regex = str(msg)
            self.assertErrorRegex(Exception, regex, logger.raiseException, msg)