def do_init(self, c):
        """init [-flag ...] PARAMETERS_NAME|DB_NAME [VAR1=VALUE1[:VAR2=VALUE2]]
        Generates a new database out of a parameters.dat
        FLAGS::: -skip-master: the new results.sqlite is created, but gets not registered in master
                 -purge:       deletes the results.sqlite db, if it already existed 
        """
        
        flags,c = self.parse_command_line(c)
        i_arg = c[0]
        
        try:
            i_arg, db_name = self.translate_name(i_arg)
        except: 
            utils.newline_msg("ERR", "results db '%s' doesn't exist. Can not init it" )
            return

        if "purge" in flags:
            try:
                os.remove(db_name)
            except:
                utils.newline_msg("WRN", "db '%s' could not be removed... skipping"%db_name)  
            self.do_deregister(i_arg)

         
        if len(c) >1: self.do_set( ":".join( c[1:] ) )
        
        parser = EnsembleCSVBuilder(stream = open(i_arg), db_name=db_name)
        parser.init_db(  )

 def __init__(self, parameter_file):
     self.parameter_file = parameter_file
     
     self.mit = spgb.MultIteratorParser( open(parameter_file) )
     self.variables = self.mit.varying_items()
     
     
     self.settings, foo =  self.get_settings(self.mit.command, "inputs")
     
     settings_output, self.output_columns = self.get_settings(self.mit.command, "stdout")
     
     newline_msg( "INF", "output columns: %s"% enumerate(self.output_columns) )
     self.settings.update(settings_output)
     
     self.x_axis =  self.variables[-1]
     x_axis_iter = self.mit.data[ self.mit.position_of( self.x_axis ) ]
     if x_axis_iter.type == "*":
         
         self.add_setting( self.x_axis, "scale = log")
         self.add_setting( self.x_axis, "lim = (%s,%s)"%(x_axis_iter.xmin,x_axis_iter.xmax))
         
         
     
     try:
         self.coalesced_vars = [ self.variables[-2] ]
     except:
         self.coalesced_vars = []
         
     try:
         self.separated_vars = self.variables[:-2]
     except:
         self.separated_vars = []

    def dump_result(self):
         """ loads the next parameter atom from a parameter ensemble"""


         #:::~ status can be either
         #:::~    'N': not run
         #:::~    'R': running
         #:::~    'D': successfully run (done)
         #:::~    'E': run but with non-zero error code

         if self.return_code != 0:
             self.query_set_run_status("E")
             return

         for line in self.output:
             table_name, output_columns = self.parse_output_line(line)
             output_columns = [self.current_spg_uid, self.current_spg_vsid, self.current_spg_rep] + output_columns

             cc = 'INSERT INTO %s (%s) VALUES (%s) ' % (table_name, ", ".join(self.table_columns[table_name]),
                                                                   ", ".join(["'%s'" % str(i) for i in output_columns]))

#             print cc
#
             try:
                 self.execute_query(cc)
                 self.query_set_run_status("D", self.current_spg_uid, self.run_time)
             except sql.OperationalError as e:
                 v = str(e).split()
                 nv, nc = int(v[0]), int(v[3])
                 utils.newline_msg("DB", "Fatal, '%d' values for the '%d' output columns %s expected " % (nv-3,nc-3, self.table_columns[table_name][3:]))
                 sys.exit(1)

    def get_flags_and_db(self, c,init_db = True):

        flags, args =  self.parse_command_line(c)
        # if len(args)== 1:
        #     utils.newline_msg("ERR", "a single file was expected or could not parse flags")
        #     return flags, args, None

        db_name = args[-1]
        args = args[:-1]


        if db_name.isdigit():
            id = int(db_name)
            rdb = self.master_db.result_dbs
            filtered = [x for x in rdb if rdb[x]['id'] == id]
            if filtered:
                db_name = filtered[0]
            else:
                utils.newline_msg("ERR", "database with id '%s' doesn't exist." % db_name)
                return flags, args.append(db_name), None
        else:
                full_name, path, base_name, extension = utils.translate_name(db_name)
                # print "do_init::: ",self.translate_name(i_arg)
                db_name = "%s/%s.spgql" % (path, base_name)
                # sim_name = "%s/%s.spg" % (path, base_name)
                if not os.path.exists(db_name):
                    utils.newline_msg("ERR", "database with name '%s' doesn't exist." % utils.shorten_name(db_name))
                    return flags, args.append(db_name), None

        return flags, args, self.EnsembleConstructor(db_name, init_db)

    def plot_all_join_outputs(self, Plotter):
        newline_msg("INF", "%s - %s - %s" % (self.separated_vars, self.coalesced_vars, self.x_axis))

        table_name = self.base_name + "_results.csv"
        ctp = Plotter(table_name)

        ctp.x_axis = self.x_axis

        ctp.y_axis = self.output_columns

        ctp.separated_vars = self.separated_vars
        ctp.coalesced_vars = self.coalesced_vars

        ctp.settings = self.settings

        plot_fname = self.base_name + "_plot"
        if len(ctp.separated_vars) > 1:
            plot_fname += "_" + "_".join(ctp.separated_vars)
        if len(ctp.separated_vars) == 1:
            plot_fname += "_" + ctp.separated_vars[0]

        plot_fname += ".pdf"

        newline_msg("OUT", plot_fname)
        ctp.plot_all_join_outputs(output_name=plot_fname)

 def do_load(self,c):
     """loads a queue as active one"""
     self.__update_queue_list()
     if len( filter(lambda x: x[1] == c, self.queues) ):
        self.current_queue = c
     else:
        newline_msg("ERR", "queue '%s' not found"%c)
     print "loaded '%s'"%self.current_queue

 def do_setup_output_column(self,c):
     """sets which columns to generate output from"""
     if not self.current_param_db:
         utils.newline_msg("WRN", "current db not set... skipping")
         return
     c = c.split(",")
     if not set(c).issubset(  self.current_param_db.output_column ):
         utils.newline_msg("ERR", "the column(s) is (are) not in the output: %s"%( set(c) - set(  self.current_param_db.output_column )) )
     self.output_column = c

 def __init__(self, stream=None, db_name = "results.sqlite", timeout = 5):
     MultIteratorParser.__init__(self, stream)
 #    print  check_params.consistency(self.command, self)
     if not check_params.consistency(self.command, self):
         utils.newline_msg("ERR","parameters.dat file is not consistent.")
         sys.exit(1)
     self.stdout_contents = check_params.contents_in_output(self.command)
             
     self.connection =  sql.connect(db_name, timeout = timeout)
     self.cursor = self.connection.cursor()

 def do_add(self, c):
     """adds queue queue"""
     self.__update_queue_list()
     
     if len( filter(lambda x: x[1] == c, self.queues) ):
             newline_msg("ERR", "queue '%s' already exists"%queue, 2)
             return 
     self.master_db.execute_query( "INSERT INTO queues (name, max_jobs, status) VALUES (?,?,?)",c,  1, 'S')
     os.makedirs("%s/queue/%s"%(VAR_PATH,c))
     self.current_queue = c

 def do_setup_output_column(self,c):
     """sets which columns to generate output from"""
     if not self.current_param_db:
         utils.newline_msg("WRN", "current db not set... it must be loaded first",2)
         return
     flags, args = self.parse_command_line(c)
     cols = args[0].split(",")
     if not set(cols).issubset(  self.current_param_db.table_columns[self.table] ):
         utils.newline_msg("ERR", "the column(s) is (are) not in the output: %s"%( set(cols) - set(  self.current_param_db.table_columns[self.table] )) )
     self.table_columns = cols

    def launch_workers(self):
        target_jobs, = self.master_db.query_master_fetchone('SELECT max_jobs FROM queues WHERE name = "default"')

        self.master_db.update_list_ensemble_dbs()
        if len(self.master_db.active_dbs) == 0:
            utils.inline_msg("MSG", "No active dbs... sleeping ")
            return

        current_count = self.active_threads()
#        print "+++++++++++", to_launch
        vec_to_launch = []

        launch = defaultdict(lambda: 0)
        running = {}

        for ae in self.master_db.active_dbs:
            ens = self.master_db.result_dbs[ae  ]
            running[ ens['id'] ] = self.active_processes[ ae ]
            qty_to_launch = int( m.floor(0.5 + target_jobs*ens['weight']/self.master_db.normalising) - self.active_processes[ ae ] )

            vec_to_launch += qty_to_launch * [ae]
            launch[ ens['id'] ] += qty_to_launch
 #       for id in launch:
 #           print "+++ (%d) %d + %d = //%d//, "%( id, launch[id], running[id],launch[id]+running[id] )
 #       print

        to_launch = len(vec_to_launch)
        if to_launch >= 0:
             utils.newline_msg("STATUS", utils.str_color( "@green[n_jobs=%d] run=%d %s ::: new=%d" % (target_jobs,current_count, dict(running),to_launch) ) )
        else:
             utils.newline_msg("STATUS", utils.str_color( "@yellow[n_jobs=%d] run=%d :!: exceeded number" % (target_jobs,current_count)) )

 #       print to_launch, len( vec_to_launch ), launch

#        for i_t in range(to_launch):
        for ae in vec_to_launch:
            pick = self.master_db.EnsembleConstructor(ae, init_db=True)
            self.lock.acquire()
#            pick = self.master_db.pick_ensemble()

            pick.test_run = self.test_run

            status = pick.get_updated_status()
            if status['process_not_run'] == 0:
                print "+D+ ----- %s " % (pick.full_name)
                self.master_db.query_master_db('UPDATE dbs SET status= ? WHERE full_name = ?', "D", pick.full_name)
                return

            self.lock.release()

            nt = SPGRunningAtom(pick, self.lock, self.active_processes)
#            nt.test_run = self.test_run
            # nt = SPGRunningAtom(pick, lock=self.get_lock( pick ) )

            nt.start()

 def do_run_script(self,c):
     """executes a script file with commands accepted in this cmdline parser"""
     if not os.path.exists(c):
         utils.newline_msg("FIL", "file doesn't exist")
         return
     for l in open(c):
         l = l.strip()
         if not l: continue 
         if l[0] == "#": continue
         
         self.onecmd(l.strip())

 def do_conf(self,c):
     """prints the current configuration"""
     if not self.current_param_db:
         utils.newline_msg("WRN", "current db not set... skipping")
         return
     print " -- db: %s"%( self.shorten_name( self.current_param_db.full_name ) )
     print "  + variables = %s "%( ", ".join(self.current_param_db.variables ) )
     print "  + entities = %s "%( ", ".join(self.current_param_db.entities ) )
     print "  + columns = %s "%( ", ".join(self.current_param_db.output_column ) )
     print "  + split_columns = %s  / raw_data = %s / restrict_by_val = %s"%(self.split_columns, self.raw_data, self.restrict_by_val)
     print "  + vars (separated-coalesced-in_table) = %s - %s - %s "%(self.current_param_db.separated_vars, self.current_param_db.coalesced_vars, self.current_param_db.in_table_vars)

 def update_ensemble_list(self):
         self.normalising = 0.
         self.active_dbs = []
         for i in self.result_dbs.keys():
             if self.result_dbs[i] == None:
                 del self.result_dbs[i]
                 utils.newline_msg("MSG", "removing db '%s' from the running list"%i)
                 continue
             if self.result_dbs[i].status == 'R':
                 self.normalising += self.result_dbs[ i ].weight
                 self.active_dbs.append( self.result_dbs[ i ] )

 def setup_separated_output(self, conf):
     try: 
         i = int(conf)
         separated = self.variables[:i+1]
     except:
         separated = conf.split(",")
     if set(separated).issubset( set(self.variables) ):
         self.separated_vars = separated
         self.coalesced_vars = [ i for i in self.variables if ( i not in self.separated_vars ) and ( i not in self.in_table_vars ) ]
     else:
         utils.newline_msg("VAR", "the variables '%s' are not recognised"%set(separated)-set(self.variables) )

 def harvest_atoms(self):
     ls_atoms = os.listdir("%s/run"%(VAR_PATH) )
     self.harvested_atoms  = len(ls_atoms)
     for i_d in ls_atoms:
         pd = ParameterAtom(i_d)
         try:
             pd.load(src = 'run')
         except:
             utils.newline_msg("WRN", "could not pickle '%s'...skipping"%i_d, 2)
             continue
         a_db =self.result_dbs[pd.full_db_name]
         pd.dump_result_in_ensemble( a_db  )

    def get_db(self, c):
        """it returns the db name (or None) of a database identified either from its id or """

        try:
                full_name, path, base_name, extension = utils.translate_name(c.strip())
                db_name = "%s/%s.spgql" % (path, base_name)
                sim_name = "%s/%s.spg" % (path, base_name)
        except:
                utils.newline_msg("ERR", "results db '%s' doesn't exist." % c)
                return None

        return self.EnsembleConstructor(db_name, init_db=True)

 def do_load(self,c):
     """loads one of the registered databases from the master"""
     c = c.split()
     if len(c) >1:
         utils.newline_msg("ERR", "only one db can be loaded at a time", 2)
         return
     ret = self.get_db_from_cmdline(c[0])
     if ret:
         self.current_param_db = ret 
         print " --- loaded db '%s'"%self.shorten_name( self.current_param_db.full_name )
     else:    
         utils.newline_msg("ERR", "db does not exist", 2)

 def do_set_as_var(self,c):
     """ Sets a (set of) non-variables as variable """
     if not self.current_param_db: 
         utils.newline_msg("WRN", "current db not set... skipping")
         return 
     ls_vars = c.split(",")
     if not set(ls_vars).issubset( set(self.current_param_db.entities) ):
         utils.newline_msg("VAR", "the variables '%s' are not recognised"%set(ls_vars)-set(self.current_param_db.entities) )
         return
     for v in ls_vars:
         self.current_param_db.execute_query( 'UPDATE entities SET varies=1 WHERE name = ?', v)
     self.current_param_db.init_db()

 def do_set(self, c):
     """sets a VAR1=VALUE1[:VAR2=VALUE2]
     sets a value in the currently loaded database """
     
     if not self.current_param_db: 
         utils.newline_msg("WRN", "current db not set... skipping")
         return 
     
     ret = utils.parse_to_dict(c, allowed_keys=self.possible_keys)
     if not ret: 
         return
     for k in ret.keys():
         self.__dict__[k] = ret[k]