def stop_processes(self):
        """Iterate through all of the consumer processes shutting them down."""
        self._set_state(self.STATE_SHUTTING_DOWN)
        LOGGER.debug('Stopping consumer processes')
        self._stop_timers()

        active_processes = multiprocessing.active_children()

        # Stop if we have no running consumers
        if not active_processes:
            LOGGER.info('All consumer processes have stopped')
            return self._set_state(self.STATE_STOPPED)

        # Iterate through all of the bindings and try and shutdown processes
        for process in active_processes:
            self._stop_process(process)

        # Wait for them to shutdown cleanly
        time.sleep(2)

        iterations = 0
        while multiprocessing.active_children():
            LOGGER.debug('Waiting on %i active processes to shut down',
                         self.total_process_count)
            time.sleep(1)
            iterations += 1

            # If the shutdown process waited long enough, kill the consumers
            if iterations == self._MAX_SHUTDOWN_WAIT:
                self._kill_processes()
                break

        LOGGER.debug('All consumer processes stopped')
        self._set_state(self.STATE_STOPPED)

def stop(processes, stop_event):
  """Stop queuing processes."""
  stop_event.set()
  for p in processes:
    if isinstance(p, mp.Process):
      p.terminate()
  mp.active_children()  # Use to join the killed processes above.

def brutePasses(userlist,passlist,hosti,pathi,porti,securei,psize,loginsi):
	global host
	host = hosti
	global port
	port = porti
	global secure
	secure = securei
	global logins
	logins = loginsi
	global path
	path = pathi
	global usernames
	usernames = userlist
	usersize = len(userlist)
	passsize = len(passlist)
	
	# manage pool
	if (psize == 0):	psize = 5
	if ((usersize*passsize) <= psize):	chunksize = 1
	else:	chunksize = (((usersize*passsize) / psize) + ((usersize*passsize) % psize))
	#print("%s" % ((ceil(float((usersize*passsize)) / psize)) + ((usersize*passsize) % psize)))
	print("Userlist size: %d\tPassword size: %d\tChunk size: %d\tPool size: %d" % (usersize,passsize,chunksize,psize))
	pool = Pool(processes=psize)
        for chunk in itertools.izip(grouper(passlist,chunksize)):  pool.map_async(worker,chunk)
        pool.close()
        try:
                while(len(active_children()) > 0): # how many active children do we have
                        sleep(2)
                        ignore = active_children()
        except KeyboardInterrupt:       exit('CTRL^C caught, exiting...\n\n')
	print("Password bruteforce attempts completed")

    def execute(self, *args, **options):
        if options['start-over']:
            init_db_for_crawl(DB_CONN_URL)

        num2crawl = options['count']
        interval = options['interval']
        if options['url']:
            # run once, test mode (no writeback)
            p = StoreDetailCrawler(interval=interval, count=num2crawl, writeback=False, url=options['url'])
            p.start()
            p.join()
        else:
            # run pool (could have pool size of 1)
            pool_size = options['pool-size']
            while (len(multiprocessing.active_children())<pool_size and not completed):
                num = pool_size - len(multiprocessing.active_children())
                for i in range(num):
                    p = StoreDetailCrawler(count=num2crawl, interval=interval)
                    p.start()
                    time.sleep(2)

                while(len(multiprocessing.active_children())>=pool_size):
                    time.sleep(0.5)
                
                print 'COMPLETED: ',completed

def fork_it(args):
    threads = int(args.t)
    childs = int(args.f)
    len_hosts = len(HOSTLIST)

    print "[*] attacking %d target(s)\n" "[*] cracking up to %d hosts parallel\n" "[*] threads per host: %d" % (
        len_hosts,
        childs,
        threads,
    )

    i = 1
    for host in HOSTLIST:
        host = host.replace("\n", "")
        print "[*] performing attacks against %s [%d/%d]" % (host, i, len_hosts)
        hostfork = multiprocessing.Process(target=thread_it, args=(host, args))
        hostfork.start()
        # checks that we have a max number of childs
        while len(multiprocessing.active_children()) >= childs:
            time.sleep(0.001)
        time.sleep(0.001)
        i += 1

    # waiting for child processes
    while multiprocessing.active_children():
        time.sleep(1)

    def GET(self, *args):
        """
        Inspect the chained requests of all chained campaigns, requires /all
        """
        if not args:
            return dumps({"results" : 'Error: No arguments were given'})
        if args[0] != 'all':
            return dumps({"results" : 'Error: Incorrect argument provided'})

        if len(multiprocessing.active_children()) < 1:  ##see if already running
            ccid_list = self.listAll()                  ##we run only 1 inspection
                              ## in background
            shuffle(ccid_list)
            try:
                p  = multiprocessing.Process(target = self.multiple_inspect,
                        args = (','.join( ccid_list ),))

                p.start()
                return dumps({"results" : True, "message":
                        "Successfully forked inspection to background. PID: %s"
                        % p.pid})

            except Exception as e:
                self.logger.error('Error while forking an inspection')
                self.logger.exception(e)
                return dumps({"results" : False,
                        message : "Failed in forking the process"})
        else:
            return dumps({"results" : True,
                    "message" : "Already running inspection in background. PID: %s"
                            %multiprocessing.active_children()[0].pid})

def bruteUser(userlist, psize, hosti, pathi, porti, securei, userfound):
    global host
    host = hosti
    global port
    port = porti
    global secure
    secure = securei
    global userout
    userout = userfound
    global path
    path = pathi
    f = open(userout, "w").close()
    usersize = len(userlist)
    # manage pool
    if usersize <= psize:
        chunksize = 1
    else:
        chunksize = (usersize / psize) + (usersize % psize)
    print("Userlist size: %d\tChunk size: %d\tPool size: %d" % (usersize, chunksize, psize))
    print("Bruteforcing usernames")
    pool = Pool(processes=psize)
    for chunk in itertools.izip(grouper(userlist, chunksize)):
        pool.map_async(worker, chunk)
    pool.close()
    try:
        while len(active_children()) > 0:  # how many active children do we have
            sleep(2)
            ignore = active_children()
    except KeyboardInterrupt:
        exit("CTRL^C caught, exiting...\n\n")
    print("Username bruteforce complete")

def find(mappe, cpu=6, minLengde=5):
    import os, time
    from multiprocessing import Process, Queue, active_children
    arbejdsliste = os.listdir(mappe)
    while arbejdsliste and len(active_children()):
        while len(active_children()) < cpu:
            #spawn child
        time.sleep(1)
    print 'Færdig med arbejdet!'
    return

def child(fil, mappe):
    #indlæs filen
    udfil = open(mappe +'/homopolymer-'+fil, 'wb')
    for sekvens in filen:
        for base in ('A', 'T', 'C', 'G'):
            ind = 0
            while 1:
                try:
                    homopolymer = sekvens[ind:].index(base)
                    udfil.write('>|' + str(homopolymer) + '|\n' + sekvens[homopolymer-100:homopolymer+100])
                    ind = homopolymer + 100
                except ValueError:
                    break
    udfil.close()
    return

if __name__ == "__main__":
    import sys
    find(sys.argv[1])
    exit()

def start_data_process(config, start_dt_arg, end_dt_arg):
    """Loop through the entity types and perform the main function """
    g_logger.info("Start processing data from %s to %s" %
                  (str(start_dt_arg), str(end_dt_arg)))

    processes = []
    for kind, fetch_intervals in config['kinds'].iteritems():
        # fetch_intervals is an array with format [int, int, bool, bool, str]
        # [Save interval, fetch interval, isMutable, is_ndb, json key]
        interval = dt.timedelta(seconds=int(fetch_intervals[0]))
        fetch_interval = fetch_intervals[1]
        start_dt = start_dt_arg
        end_dt = end_dt_arg
        while start_dt < end_dt:
            if len(active_children()) < config['max_threads']:
                next_dt = min(start_dt + interval, end_dt)
                p = Process(target=fetch_and_process_data,
                    args=(kind, start_dt, next_dt, fetch_interval, config))
                p.start()
                download_params = {"kind": kind, "start_dt": start_dt,
                                   "end_dt": next_dt, "start": time.time()}
                processes.append((p, download_params))
                start_dt = next_dt
            else:
                monitor(config, processes)
            # wait for 2 secs to space out the queries
            time.sleep(2)
    while len(active_children()) > 0:
        monitor(config, processes)
        time.sleep(10)

    def manager_thread_main(self):
        """ Checks for workers that died unexpectedly and listens to their
            status update messages.
        """
        while True:
            # wait up to 60 seconds
            try:
                command, argument = self.manager_thread_queue.get(True, 60)

                if command == 'quit':
                    break
                elif command == 'start':
                    with self._lock:
                        self.now_building[argument] = True
                elif command == 'end':
                    with self._lock:
                        self.now_building[argument] = False
                else:
                    logger.warn("Unknown command to manager thread: %s" % command)

            except Queue.Empty:
                pass

            # this one is to remove zombie processes
            multiprocessing.active_children()

            with self._lock:
                ids_to_restart = []
                for id, worker in self.workers.items():
                    if not worker.is_alive():
                        ids_to_restart.append(id)

                for id in ids_to_restart:
                    self.restart(id)

    def run_tasks(self, tasks):
        # Join whatever children are still sitting around
        multiprocessing.active_children()

        queue = multiprocessing.Queue()
        execute_ps = multiprocessing.Process(
            target=self._execute, args=[tasks, queue])
        execute_ps.start()

        def generate_response(execute_ps, queue):
            while execute_ps.is_alive():
                try:
                    data = queue.get_nowait()
                    yield data
                except Queue.Empty:
                    time.sleep(1)

            execute_ps.join()

            # suck the last goodness out of the queue before moving on
            while True:
                try:
                    data = queue.get_nowait()
                    yield data
                except Queue.Empty:
                    break

        return execute_ps, threadsafe_iter(
            generate_response(execute_ps, queue))

    def CheckIfStillUsed(self):
        import time
        if self.LoadItemFromSessionStore('status', 'processID') == None:
            return

        # if a new process ID is in the session data, another process was started and this process was abandoned
        if self.LoadItemFromSessionStore('status', 'processID') != os.getpid() and self.LoadItemFromSessionStore('status', 'processID') != 0:
            print "**** SMLRPP Exiting on process ID, session_status['processID'] = ", self.LoadItemFromSessionStore('status', 'processID'), " os.getpid() = ", os.getpid()
            sys.stdout.flush()

            time.sleep(1.0)
            if self.pool:
                self.pool.close()
                self.pool.join()
                self.pool = None
            for p in multiprocessing.active_children():
                p.terminate()
            os._exit(0) # kills pool processes

        # if the status has not been checked in the past 30 seconds, this process was abandoned
        if (time.time() - self.LoadItemFromSessionStore('status', 'time_of_last_status_check')) > 300:
            print "**** SMLRPP Exiting on time of last status check"
            sys.stdout.flush()

            time.sleep(1.0)
            if self.pool:
                self.pool.close()
                self.pool.join()
                self.pool = None
            for p in multiprocessing.active_children():
                p.terminate()
            os._exit(0) # kills pool processes

    def shutdown(self, c):
        '''
        Shutdown this process
        '''
        try:
            try:
                util.debug('manager received shutdown message')
                c.send(('#RETURN', None))

                if sys.stdout != sys.__stdout__:
                    util.debug('resetting stdout, stderr')
                    sys.stdout = sys.__stdout__
                    sys.stderr = sys.__stderr__

                util._run_finalizers(0)

                for p in active_children():
                    util.debug('terminating a child process of manager')
                    p.terminate()

                for p in active_children():
                    util.debug('terminating a child process of manager')
                    p.join()

                util._run_finalizers()
                util.info('manager exiting with exitcode 0')
            except:
                import traceback
                traceback.print_exc()
        finally:
            exit(0)

    def shutdown(self):
        BuiltinCore.shutdown(self)
        self.logger.info("Closing RPC command queues")
        self.rpc_q.close()

        def term_children():
            """ Terminate all remaining multiprocessing children. """
            for child in multiprocessing.active_children():
                self.logger.error("Waited %s seconds to shut down %s, "
                                  "terminating" % (self.shutdown_timeout,
                                                   child.name))
                child.terminate()

        timer = threading.Timer(self.shutdown_timeout, term_children)
        timer.start()
        while len(multiprocessing.active_children()):
            self.logger.info("Waiting for %s child(ren): %s" %
                             (len(multiprocessing.active_children()),
                              [c.name
                               for c in multiprocessing.active_children()]))
            time.sleep(1)
        timer.cancel()
        self.logger.info("All children shut down")

        while len(threading.enumerate()) > 1:
            threads = [t for t in threading.enumerate()
                       if t != threading.current_thread()]
            self.logger.info("Waiting for %s thread(s): %s" %
                             (len(threads), [t.name for t in threads]))
            time.sleep(1)
        self.logger.info("Shutdown complete")

    def ValidateProxies(self, proxyList):
            
        maxProc = 50
        
        tests = ["http://www.baidu.com"]
    
        result = Queue()
       
        start = time.clock()
        
        for i in proxyList:
            p = Process(target=self.CheckProxy, args=(i, tests, result))
            p.start()  
            
            if len(multiprocessing.active_children()) > maxProc:
                #print('active_children: ', multiprocessing.active_children())
                p.join()
            
        while len(multiprocessing.active_children()) > 0:
            time.sleep(3)
        end = time.clock()
        #print("total time for validation:", end - start, "s")
        
        self.pool = []
        
        for i in range(result.qsize()):
            a = result.get()
            self.pool += [Proxy(a[0], a[1])]

        
        print("{0} validated".format(len(self.pool)))

def main_loop(config):
    logger.info(
        u'Run main loop. Worker pool size={}. Sleep time is {}.'.format(
            config.WORKER_POOL_SIZE, config.SLEEP
        ))
    parent_pid = os.getpid()
    while True:
        if check_network_status(config.CHECK_URL, config.HTTP_TIMEOUT):
            required_workers_count = config.WORKER_POOL_SIZE - len(
                active_children())
            if required_workers_count > 0:
                logger.info(
                    'Spawning {} workers'.format(required_workers_count))
                spawn_workers(
                    num=required_workers_count,
                    target=worker,
                    args=(config,),
                    parent_pid=parent_pid
                )
        else:
            logger.critical('Network is down. stopping workers')
            for c in active_children():
                c.terminate()

        sleep(config.SLEEP)

 def listener(self, pname, recv_data):
     ip, port = pname.split('-')
     #client = pymongo.MongoClient()
     #collection = client.minerva.sensors
     #print('starting receiver')
     KEYFILE = self.config['Event_Receiver']['certs']['private_key']
     CERTFILE = self.config['Event_Receiver']['certs']['server_cert']
     s = socket(AF_INET, SOCK_STREAM)
     s.bind((ip, int(port)))
     s.listen(5)
     s_ssl = ssl.wrap_socket(s, keyfile=KEYFILE, certfile=CERTFILE, server_side=True, ssl_version=ssl.PROTOCOL_SSLv3)
     active_recv = []
     while True:
         try:
             for p in active_recv:
                 if p not in active_children():
                     p.join()
                     active_recv.remove(p)
             if len(active_children()) < int(self.max_threads):
                 #print('accepting connections')
                 c, a = s_ssl.accept()
                 #print('Got connection', c, a)
                 pr = Process(target=recv_data, args=((a[0], c)))
                 pr.start()
                 active_recv.append(pr)
             else:
                 #print('sleeping')
                 time.sleep(.001)
         except Exception as e:
             print('{}: {}'.format(e.__class__.__name__,e))

    def start(self):
        log = self.logger
        log.info("Agent is starting")
        while True:
            if self.is_registered():
                log.debug("Checking if there are any new job orders")
                job_order = self.get_job_order()
                if job_order:
                    log.info("Received job order %s" % json.dumps(job_order, indent=4, sort_keys=True))
                    job = twindb_agent.job.Job(job_order)
                    proc = multiprocessing.Process(target=job.process,
                                                   name="%s-%s" % (job_order["type"], job_order["job_id"]))
                    proc.start()
                    # Dispatcher can't handle parallel jobs. Will wait till job finishes.
                    # After the bug is fixed .join() should be removed
                    # https://bugs.launchpad.net/twindb/+bug/1484342
                    proc.join()

                # Report replication status
                log.debug("Reporting replication status")
                proc = multiprocessing.Process(target=twindb_agent.handlers.report_show_slave_status,
                                               name="report_sss")
                proc.start()

                # Report agent privileges
                log.debug("Reporting agent granted privileges")
                proc = multiprocessing.Process(target=twindb_agent.handlers.report_agent_privileges,
                                               name="report_agent_privileges")
                proc.start()

                # Calling this has the side affect of "joining" any processes which have already finished.
                multiprocessing.active_children()
            else:
                log.warn("This agent(%s) isn't registered" % self.config.server_id)
            time.sleep(self.config.check_period)

 def start (self):
     delete_break_file ()
     enddate = self.last_trade_date ()
     startdate = self.get_start_date (enddate, 100)
     # 分配每个线程的代码任务列表
     worker_count = mp.cpu_count () * 5
     task_queue = mp.Queue ()
     # 创建和启动工作进程
     progress = mp.Value ('i', 0)
     workers = []
     for i in xrange (worker_count):
         w = Monitor (task_queue, progress, startdate, enddate)
         workers.append (w)
         w.start ()
     print "start %d workers to run" % len (workers)
     sys.stdout.flush ()
     # 往任务队列添加任务
     for code in self.code_list:
         task_queue.put (code)
     for i in xrange (worker_count + 10):
         task_queue.put (None)
     # 等待执行完毕
     while progress.value < len (self.code_list):
         time.sleep (10)
         print "-------->workers(%d) progress:%d => %d <---------" % (len (mp.active_children ()), progress.value, len (self.code_list))
         sys.stdout.flush ()
         if len (mp.active_children ()) <= 0:
             print "All Workers Exits"
             sys.stdout.flush ()
             break

def main(): 
    global badExtensionCounter, failedFlag, pool, failedProcessCounter#, db
    
    sql_setup() # Set-up SQL Database/check to see if exists
    
    # Initiate File Path Handler
    observer = Observer()
    observer.schedule(MyHandler(), path=file_path, recursive=True)
    observer.start()
    
    cpuCount = multiprocessing.cpu_count() # Count all available CPU's
    print "\nTotal CPU Count: %d"%(cpuCount)
    pool = multiprocessing.Pool(4, worker,(processQueue,)) # Create 4 child processes to handle all queued elements
    active = multiprocessing.active_children() # All active child processes
    print "Total number of active child processes: %s\n"%(str(active))
    
    try:
        while True:
            time.sleep(0.2)
    except KeyboardInterrupt:
        pool.terminate() # Stop all child processes
        pool.join() # Join the processes with parent and terminate
        active = multiprocessing.active_children() # All active child processes, list should be empty at this point.
        print "\nTotal number of active child processes: %s\n"%(str(active))
        shutdown() # Run shutdown sequence        
        observer.stop()
        observer.join()
        sys.exit(1)