def testClushConfigSetRlimitValueError(self):
        """test CLI.Config.ClushConfig (setrlimit ValueError)"""
        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
        f = tempfile.NamedTemporaryFile(prefix='testclushconfig')
        f.write(dedent("""
            [Main]
            fanout: 42
            connect_timeout: 14
            command_timeout: 0
            history_size: 100
            color: auto
            # Use wrong fd_max value to generate ValueError
            fd_max: -1
            verbosity: 1"""))
        f.flush()
        parser = OptionParser("dummy")
        parser.install_config_options()
        parser.install_display_options(verbose_options=True)
        parser.install_connector_options()
        options, _ = parser.parse_args([])
        config = ClushConfig(options, filename=f.name)
        f.close()
        display = Display(options, config)

        class TestException(Exception): pass

        def mock_vprint_err(level, message):
            if message.startswith('Warning: Failed to set max open files'):
                raise TestException()

        display.vprint_err = mock_vprint_err
        self.assertRaises(TestException, set_fdlimit, config.fd_max, display)

        soft2, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
        self.assertEqual(soft, soft2)

    def testClushConfigSetRlimit(self):
        """test CLI.Config.ClushConfig (setrlimit)"""
        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
        hard2 = min(32768, hard)
        f = tempfile.NamedTemporaryFile(prefix='testclushconfig')
        f.write("""
[Main]
fanout: 42
connect_timeout: 14
command_timeout: 0
history_size: 100
color: auto
fd_max: %d
verbosity: 1
""" % hard2)

        f.flush()
        parser = OptionParser("dummy")
        parser.install_display_options(verbose_options=True)
        parser.install_connector_options()
        options, _ = parser.parse_args([])
        config = ClushConfig(options, filename=f.name)
        self.assert_(config != None)
        display = Display(options, config)
        self.assert_(display != None)

        # force a lower soft limit
        resource.setrlimit(resource.RLIMIT_NOFILE, (hard2/2, hard))
        # max_fdlimit should increase soft limit again
        set_fdlimit(config.fd_max, display)
        # verify
        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
        self.assertEqual(soft, hard2)
        f.close()

 def __init__(self, max_threads=None):
     '''
     If max_threads is not supplied, calculates a reasonable value based
     on system resource limits.
     '''
     self.active_requests = set()
     if not max_threads:
         # man getrlimit: "RLIMIT_NPROC The maximum number of processes (or,
         # more precisely on Linux, threads) that can be created for the
         # real user ID of the calling process."
         try:
             import resource
             rlimit_nproc = resource.getrlimit(resource.RLIMIT_NPROC)[0]
             rlimit_nofile = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
             max_threads = min(rlimit_nofile // 10, rlimit_nproc // 2)
             # resource.RLIM_INFINITY == -1 which can result in max_threads == 0
             if max_threads <= 0 or max_threads > 5000:
                 max_threads = 5000
             self.logger.info(
                     "max_threads=%s (rlimit_nproc=%s, rlimit_nofile=%s)",
                     max_threads, rlimit_nproc, rlimit_nofile)
         except Exception as e:
             self.logger.warn(
                     "unable to calculate optimal number of threads based "
                     "on resource limits due to %s", e)
             max_threads = 100
             self.logger.info("max_threads=%s", max_threads)
     self.max_threads = max_threads
     self.pool = concurrent.futures.ThreadPoolExecutor(max_threads)

    def __init__(
        self,
        application,
        environ=None,
        bindAddress=None,
        umask=None,
        multiplexed=False,
        debug=False,
        roles=(FCGI_RESPONDER,),
        forceCGI=False,
        timeout=None,
        **kw
    ):
        """
        environ, if present, must be a dictionary-like object. Its
        contents will be copied into application's environ. Useful
        for passing application-specific variables.

        bindAddress, if present, must either be a string or a 2-tuple. If
        present, run() will open its own listening socket. You would use
        this if you wanted to run your application as an 'external' FastCGI
        app. (i.e. the webserver would no longer be responsible for starting
        your app) If a string, it will be interpreted as a filename and a UNIX
        socket will be opened. If a tuple, the first element, a string,
        is the interface name/IP to bind to, and the second element (an int)
        is the port number.
        """
        BaseFCGIServer.__init__(
            self,
            application,
            environ=environ,
            multithreaded=False,
            multiprocess=True,
            bindAddress=bindAddress,
            umask=umask,
            multiplexed=multiplexed,
            debug=debug,
            roles=roles,
            forceCGI=forceCGI,
        )
        for key in ("multithreaded", "multiprocess", "jobClass", "jobArgs"):
            if kw.has_key(key):
                del kw[key]
        PreforkServer.__init__(self, jobClass=self._connectionClass, jobArgs=(self, timeout), **kw)

        try:
            import resource

            # Attempt to glean the maximum number of connections
            # from the OS.
            try:
                maxProcs = resource.getrlimit(resource.RLIMIT_NPROC)[0]
                maxConns = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
                maxConns = min(maxConns, maxProcs)
            except AttributeError:
                maxConns = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
        except ImportError:
            maxConns = 100  # Just some made up number.
        maxReqs = maxConns
        self.capability = {FCGI_MAX_CONNS: maxConns, FCGI_MAX_REQS: maxReqs, FCGI_MPXS_CONNS: 0}

def raise_nofile(nofile_atleast: int = 4096) -> Tuple[int, int]:
    """
    sets nofile soft limit to at least 4096, useful for running matlplotlib/seaborn on
    parallel executing plot generators vs. Ubuntu 16.04 default ulimit -n 1024 or OS X El Captian 256
    temporary setting extinguishing with Python session.
    """
# %% (0) what is current ulimit -n setting?
    soft, ohard = res.getrlimit(res.RLIMIT_NOFILE)
    hard = ohard
# %% (1) increase limit (soft and even hard) if needed
    if soft < nofile_atleast:
        soft = nofile_atleast

        if hard < soft:
            hard = soft

        print('setting soft & hard ulimit -n {} {}'.format(soft, hard))
        try:
            res.setrlimit(res.RLIMIT_NOFILE, (soft, hard))
        except (ValueError, res.error):
            try:
                hard = soft
                print(
                    'trouble with max limit, retrying with soft,hard {},{}'.format(soft, hard))
                res.setrlimit(res.RLIMIT_NOFILE, (soft, hard))
            except Exception:
                print('failed to set ulimit, giving up')
                soft, hard = res.getrlimit(res.RLIMIT_NOFILE)

    return soft, hard

def test_rlimits():
    for k, v in sorted(resource.__dict__.items()):
        if k.startswith('RLIMIT_'):
            val = resource.getrlimit(v)
            logging.info('%s: %s', k, val)

    limits = resource.getrlimit(resource.RLIMIT_NOFILE)
    maxfiles = min(limits[1], 8192)
    logging.info('Trying to open %d files..', maxfiles)
    if real_run():
        i = 0
        try:
            # list is needed to keep files open (prevent GC)
            handles = []
            for i in range(maxfiles):
                fd = open('/tmp/file-{}'.format(i), 'w')
                fd.write('1')
                handles.append(fd)
                if i > 0 and i % 1000 == 0:
                    logging.debug('Opened %d files', i)
        except IOError:
            logging.exception('Could open %s files', i)
        for i in range(maxfiles):
            try:
                os.unlink('/tmp/file-{}'.format(i))
            except:
                pass

    def process_limit(self):
        # If our parent changed sucide
        if self.ppid != os.getppid():
            _logger.info("Worker (%s) Parent changed", self.pid)
            self.alive = False
        # check for lifetime
        if self.request_count >= self.request_max:
            _logger.info("Worker (%d) max request (%s) reached.", self.pid, self.request_count)
            self.alive = False
        # Reset the worker if it consumes too much memory (e.g. caused by a memory leak).
        rss, vms = memory_info(psutil.Process(os.getpid()))
        if vms > config['limit_memory_soft']:
            _logger.info('Worker (%d) virtual memory limit (%s) reached.', self.pid, vms)
            self.alive = False      # Commit suicide after the request.

        # VMS and RLIMIT_AS are the same thing: virtual memory, a.k.a. address space
        soft, hard = resource.getrlimit(resource.RLIMIT_AS)
        resource.setrlimit(resource.RLIMIT_AS, (config['limit_memory_hard'], hard))

        # SIGXCPU (exceeded CPU time) signal handler will raise an exception.
        r = resource.getrusage(resource.RUSAGE_SELF)
        cpu_time = r.ru_utime + r.ru_stime
        def time_expired(n, stack):
            _logger.info('Worker (%d) CPU time limit (%s) reached.', self.pid, config['limit_time_cpu'])
            # We dont suicide in such case
            raise Exception('CPU time limit exceeded.')
        signal.signal(signal.SIGXCPU, time_expired)
        soft, hard = resource.getrlimit(resource.RLIMIT_CPU)
        resource.setrlimit(resource.RLIMIT_CPU, (cpu_time + config['limit_time_cpu'], hard))

    def __set_max_open_files(self):
        # Let's check to see how our max open files(ulimit -n) setting is
        mof_s, mof_h = resource.getrlimit(resource.RLIMIT_NOFILE)
        log.info(
            'Current values for max open files soft/hard setting: '
            '{0}/{1}'.format(
                mof_s, mof_h
            )
        )
        # Let's grab, from the configuration file, the value to raise max open
        # files to
        mof_c = self.opts['max_open_files']
        if mof_c > mof_h:
            # The configured value is higher than what's allowed
            log.warning(
                'The value for the \'max_open_files\' setting, {0}, is higher '
                'than what the user running salt is allowed to raise to, {1}. '
                'Defaulting to {1}.'.format(mof_c, mof_h)
            )
            mof_c = mof_h

        if mof_s < mof_c:
            # There's room to raise the value. Raise it!
            log.warning('Raising max open files value to {0}'.format(mof_c))
            resource.setrlimit(resource.RLIMIT_NOFILE, (mof_c, mof_h))
            mof_s, mof_h = resource.getrlimit(resource.RLIMIT_NOFILE)
            log.warning(
                'New values for max open files soft/hard values: '
                '{0}/{1}'.format(mof_s, mof_h)
            )

def limit_resource():
   megs=1000000
   rsrc = resource.RLIMIT_AS
   soft, hard = resource.getrlimit(rsrc)
   print 'Soft limit starts as  :', soft
   resource.setrlimit(rsrc, (8*megs*1024, 8*megs*1024)) #limit to 8  Gigabytes
   soft, hard = resource.getrlimit(rsrc)
   print 'Soft limit changed to :', soft

def run(configs, optimal=True, final_config=None, final_config_builder=None,
        timeout=None):
    options, extra_args = parse_args()
    plan_file = options.plan_file

    # Time limits are either positive values in seconds or -1 (unlimited).
    soft_time_limit, hard_time_limit = resource.getrlimit(resource.RLIMIT_CPU)
    print 'External time limit:', hard_time_limit
    if (hard_time_limit >= 0 and timeout is not None and
        timeout != hard_time_limit):
        sys.stderr.write("The externally set timeout (%d) differs from the one "
                         "in the portfolio file (%d). Is this expected?\n" %
                         (hard_time_limit, timeout))
    # Prefer limits in the order: externally set, from portfolio file, default.
    if hard_time_limit >= 0:
        timeout = hard_time_limit
    elif timeout is None:
        sys.stderr.write("No timeout has been set for the portfolio so we take "
                         "the default of %ds.\n" % DEFAULT_TIMEOUT)
        timeout = DEFAULT_TIMEOUT
    print 'Internal time limit:', timeout

    # Memory limits are either positive values in Bytes or -1 (unlimited).
    soft_mem_limit, hard_mem_limit = resource.getrlimit(resource.RLIMIT_AS)
    print 'External memory limit:', hard_mem_limit
    memory = hard_mem_limit - BYTES_FOR_PYTHON
    # Do not limit memory if the previous limit was very low or unlimited.
    if memory < 0:
        memory = None
    print 'Internal memory limit:', memory

    assert len(extra_args) == 3, extra_args
    sas_file = extra_args.pop(0)
    assert extra_args[0] in ["unit", "nonunit"], extra_args
    unitcost = extra_args.pop(0)
    assert extra_args[0][-1] in ["1", "2", "4"], extra_args
    planner = extra_args.pop(0)

    safe_unlink("plan_numbers_and_cost")

    remaining_time_at_start = float(timeout)
    try:
        for line in open("elapsed.time"):
            if line.strip():
                remaining_time_at_start -= float(line)
    except EnvironmentError:
        print "WARNING! elapsed_time not found -- assuming full time available."

    print "remaining time at start: %s" % remaining_time_at_start

    if optimal:
        exitcodes = run_opt(configs, planner, sas_file, plan_file,
                            remaining_time_at_start, memory)
    else:
        exitcodes = run_sat(configs, unitcost, planner, sas_file, plan_file,
                            final_config, final_config_builder,
                            remaining_time_at_start, memory)
    sys.exit(_generate_exitcode(exitcodes))

    def _run(self):
        try:
            lim = resource.getrlimit(resource.RLIMIT_NOFILE)
            resource.setrlimit(resource.RLIMIT_NOFILE, (lim[1], lim[1]))

            lim = resource.getrlimit(resource.RLIMIT_NOFILE)
            print " maximum number of open files set to", lim[0]

        except Exception, e:
            print " failed to raise the maximum number of open files:", str(e)

	def prepare_subprocess():
		# create a new session for the spawned subprocess using os.setsid,
		# so we can later kill it and all children on timeout, taken from http://stackoverflow.com/questions/4789837/how-to-terminate-a-python-subprocess-launched-with-shell-true
		os.setsid()
		# Limit the size of files created during execution
		resource.setrlimit(resource.RLIMIT_NOFILE,(128,128))
		if fileseeklimit is not None:
			resource.setrlimit(resource.RLIMIT_FSIZE,(fileseeklimitbytes, fileseeklimitbytes))
			if resource.getrlimit(resource.RLIMIT_FSIZE) != (fileseeklimitbytes, fileseeklimitbytes):
				raise ValueError(resource.getrlimit(resource.RLIMIT_FSIZE))

    def set_resource_utilization_limits(self):
        if not self.memory_limit or self.memory_limit == 0:
            return

        # get the soft and hard limits for the heap limit - this is default
        soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_AS)
        resource.setrlimit(resource.RLIMIT_AS, (self.memory_limit, hard_limit))

        soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_DATA)
        resource.setrlimit(resource.RLIMIT_DATA, (self.memory_limit, hard_limit))

 def _prevent_core_dump(cls):
     """Prevent the process from generating a core dump."""
     try:
         # Try to get the current limit
         resource.getrlimit(resource.RLIMIT_CORE)
     except ValueError:
         # System doesn't support the RLIMIT_CORE resource limit
         return
     else:
         # Set the soft and hard limits for core dump size to zero
         resource.setrlimit(resource.RLIMIT_CORE, (0, 0))

def run(portfolio, executable, sas_file, plan_manager):
    attributes = get_portfolio_attributes(portfolio)
    configs = attributes["CONFIGS"]
    optimal = attributes["OPTIMAL"]
    final_config = attributes.get("FINAL_CONFIG")
    final_config_builder = attributes.get("FINAL_CONFIG_BUILDER")
    timeout = attributes.get("TIMEOUT")

    # Time limits are either positive values in seconds or -1 (unlimited).
    soft_time_limit, hard_time_limit = resource.getrlimit(resource.RLIMIT_CPU)
    print("External time limits: %d, %d" % (soft_time_limit, hard_time_limit))
    external_time_limit = None
    if soft_time_limit != resource.RLIM_INFINITY:
        external_time_limit = soft_time_limit
    elif hard_time_limit != resource.RLIM_INFINITY:
        external_time_limit = hard_time_limit
    if (external_time_limit is not None and
            timeout is not None and
            timeout != external_time_limit):
        print("The externally set timeout (%d) differs from the one "
              "in the portfolio file (%d). Is this expected?" %
              (external_time_limit, timeout), file=sys.stderr)
    # Prefer limits in the order: external soft limit, external hard limit,
    # from portfolio file, default.
    if external_time_limit is not None:
        timeout = external_time_limit
    elif timeout is None:
        print("No timeout has been set for the portfolio so we take "
              "the default of %ds." % DEFAULT_TIMEOUT, file=sys.stderr)
        timeout = DEFAULT_TIMEOUT
    print("Internal time limit: %d" % timeout)

    # Memory limits are either positive values in Bytes or -1 (unlimited).
    soft_mem_limit, hard_mem_limit = resource.getrlimit(resource.RLIMIT_AS)
    print("External memory limits: %d, %d" % (soft_mem_limit, hard_mem_limit))
    if hard_mem_limit == resource.RLIM_INFINITY:
        memory = None
    else:
        memory = hard_mem_limit
    print("Internal memory limit: %s" % memory)

    remaining_time_at_start = float(timeout) - get_elapsed_time()
    print("remaining time at start: %.2f" % remaining_time_at_start)

    if optimal:
        exitcodes = run_opt(configs, executable, sas_file, plan_manager,
                            remaining_time_at_start, memory)
    else:
        exitcodes = run_sat(configs, executable, sas_file, plan_manager,
                            final_config, final_config_builder,
                            remaining_time_at_start, memory)
    exitcode = generate_exitcode(exitcodes)
    if exitcode != 0:
        raise subprocess.CalledProcessError(exitcode, ["run-portfolio", portfolio])

    def setUp(self):
        limits = resource.getrlimit(resource.RLIMIT_NOFILE)
        print 'NOFILE rlimit:', limits
        resource.setrlimit(resource.RLIMIT_NOFILE, (10, limits[1]))
        print 'NOFILE rlimit:', resource.getrlimit(resource.RLIMIT_NOFILE)

        self.mypath = os.path.dirname(__file__)
        self.srcCat = afwTable.SourceCatalog.readFits(
            os.path.join(self.mypath, "v695833-e0-c000.xy.fits"))
        # The .xy.fits file has sources in the range ~ [0,2000],[0,4500]
        self.bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(2048, 4612)) # approximate

def setProcessLimits():
    # Read the config file
    config = ConfigParser.ConfigParser()
    config.read('config.cfg')
    sandboxingConfig = ConfigSectionMap(config, "SandboxingParams")

    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
    resource.setrlimit(resource.RLIMIT_NOFILE, (int(sandboxingConfig['nfile']), hard))
    soft, hard = resource.getrlimit(resource.RLIMIT_NPROC)
    resource.setrlimit(resource.RLIMIT_NPROC, (int(sandboxingConfig['nproc']), hard))
    soft, hard = resource.getrlimit(resource.RLIMIT_STACK)
    resource.setrlimit(resource.RLIMIT_STACK, (int(sandboxingConfig['stacksize']), hard))

def main():
    global __file__
    __file__ = os.path.abspath(__file__)
    print __file__
    if os.path.islink(__file__):
        print True
        __file__ = getattr(os, 'readlink', lambda x: x)(__file__)
    print os.getpid()
    print os.getppid()
    print resource.getrlimit(resource.RLIMIT_NOFILE)
    resource.setrlimit(resource.RLIMIT_NOFILE, (8192, -1))
    print resource.getrlimit(resource.RLIMIT_NOFILE)

def _setup_resources():
    """Attempt to increase resource limits up to hard limits.

    This allows us to avoid out of file handle limits where we can
    move beyond the soft limit up to the hard limit.
    """
    target_procs = 10240
    cur_proc, max_proc = resource.getrlimit(resource.RLIMIT_NPROC)
    target_proc = min(max_proc, target_procs) if max_proc > 0 else target_procs
    resource.setrlimit(resource.RLIMIT_NPROC, (max(cur_proc, target_proc), max_proc))
    cur_hdls, max_hdls = resource.getrlimit(resource.RLIMIT_NOFILE)
    target_hdls = min(max_hdls, target_procs) if max_hdls > 0 else target_procs
    resource.setrlimit(resource.RLIMIT_NOFILE, (max(cur_hdls, target_hdls), max_hdls))

def limit_info(line):
    rsrc = resource.RLIMIT_AS
    soft, hard = resource.getrlimit(rsrc)
    print 'Soft limit starts as:', soft
    print 'Hard limit starts as:', hard
    if line:
        limit_mb = int(line)
        limit_kb = int(line)*1024
        print 'Setting limit to %s Mb' % limit_mb
        resource.setrlimit(rsrc, (limit_kb, hard)) #limit to one kilobyte
        soft, hard = resource.getrlimit(rsrc)
        print 'Soft limit is now:', soft
        print 'Hard limit is now:', hard