def test_init(self):
     # let's add some uncertainties to this
     uncs = [RealParameter("a", 0, 1),
            RealParameter("b", 0, 1)]
     outcomes = [TimeSeriesOutcome("test")]
     constraints = []
     callback = DefaultCallback(uncs, [], outcomes, constraints,
                                nr_experiments=100)
     
     self.assertEqual(callback.i, 0)
     self.assertEqual(callback.nr_experiments, 100)
     self.assertEqual(callback.cases.shape[0], 100)
     self.assertEqual(callback.outcomes, [o.name for o in outcomes])
     
     names = rf.get_names(callback.cases.dtype)
     names = set(names)
     self.assertEqual(names, {'a', 'b', 'policy', 'model', 'scenario_id'})
     self.assertEqual(callback.results, {})
     
     # with levers
     levers = [RealParameter('c', 0, 10)]
     
     callback = DefaultCallback(uncs, levers, outcomes, constraints, 
                                nr_experiments=100)
     
     self.assertEqual(callback.i, 0)
     self.assertEqual(callback.nr_experiments, 100)
     self.assertEqual(callback.cases.shape[0], 100)
     self.assertEqual(callback.outcomes, [o.name for o in outcomes])
     
     names = rf.get_names(callback.cases.dtype)
     names = set(names)
     self.assertEqual(names, {'a', 'b', 'c','policy', 'model', 'scenario_id'})
     self.assertEqual(callback.results, {})

def get_univariate_feature_scores(x,y, score_func=F_CLASSIFICATION):
    '''
    
    calculate feature scores using univariate statistical tests. In case of
    categorical data, chi square or the Anova F value is used. In case of 
    continuous data the Anova F value is used. 
    
    Parameters
    ----------
    x : structured array
    y : 1D nd.array
    score_func : {F_CLASSIFICATION, F_REGRESSION, CHI2}
                the score function to use, one of f_regression (regression), or  
                f_classification or chi2 (classification). 
    Returns
    -------
    list of tuples 
        sorted in descending order of tuples with uncertainty and feature 
        scores (i.e. p values in this case).
    
    
    '''
    uncs = recfunctions.get_names(x.dtype)
    
    x = _prepare_experiments(x)
    
    pvalues = score_func(x, y)[1]
    pvalues = np.asarray(pvalues)

    pvalues = zip(uncs, pvalues)
    pvalues = list(pvalues)
    pvalues.sort(key=itemgetter(1))
    return pvalues

def _make_box(x):
    '''
    Make a box that encompasses all the data
    
    Parameters
    ----------
    x : structured numpy array
    
    
    '''
    
    box = np.zeros((2, ), x.dtype)
    
    names = recfunctions.get_names(x.dtype)
    
    for name in names:
        dtype = x.dtype.fields.get(name)[0] 
        mask = np.ma.getmaskarray(x[name])
        values = x[name][mask==False]
        
        if dtype == 'object':
            try:
                values = set(values)
                box[name][:] = values
            except TypeError as e:
                ema_logging.warning("{} has unhashable values".format(name))
                raise e
        else:
            box[name][0] = np.min(values, axis=0) 
            box[name][1] = np.max(values, axis=0)    
    return box  

def setup_cart(results, classify, incl_unc=[], mass_min=0.05):
    """helper function for performing cart
    
    Parameters
    ----------
    results : tuple of structured array and dict with numpy arrays
              the return from :meth:`perform_experiments`.
    classify : string, function or callable
               either a string denoting the outcome of interest to 
               use or a function. 
    incl_unc : list of strings
    mass_min : float
    
    
    Raises
    ------
    TypeError 
        if classify is not a string or a callable.
    
    """
    
    if not incl_unc:
        x = np.ma.array(results[0])
    else:
        drop_names = set(recfunctions.get_names(results[0].dtype))-set(incl_unc)
        x = recfunctions.drop_fields(results[0], drop_names, asrecarray = True)
    if type(classify)==types.StringType:
        y = results[1][classify]
    elif callable(classify):
        y = classify(results[1])
    else:
        raise TypeError("unknown type for classify")
    
    return CART(x, y, mass_min)

def _prepare_experiments(experiments):
    '''
    transform the experiments structured array into a numpy array.

    Parameters
    ----------
    experiments : structured array
    
    Returns
    -------
    ndarray
    
    '''
    uncs = recfunctions.get_names(experiments.dtype)

    temp_experiments = np.zeros((experiments.shape[0], len(uncs)))
    
    for i, u in enumerate(uncs):
        try: 
            temp_experiments[:,i] = experiments[u].astype(np.float)
        except ValueError:
            
            data = experiments[u]
            entries = sorted(list(set(data)))
            
            for j, entry in enumerate(entries):
                temp_experiments[data==entry,i] = j
    
    return temp_experiments

 def test_store_cases(self):
     nr_experiments = 3
     uncs = [ParameterUncertainty((0,1), "a"),
            ParameterUncertainty((0,1), "b"),
            CategoricalUncertainty([0, 1, 2], "c"),
            ParameterUncertainty((0,1), "d", integer=True),]
     outcomes = [Outcome("test", time=True)]
     case = {unc.name:random.random() for unc in uncs}
     case["c"] = int(round(case["c"]*2))
     case["d"] = int(round(case["d"]))
     policy = {'name':'none'}
     name = "test"
  
     callback = DefaultCallback(uncs, 
                                [outcome.name for outcome in outcomes], 
                                nr_experiments=nr_experiments,
                                reporting_interval=1)
     result = {outcomes[0].name: 1}
     callback(0, case, policy, name, result)
      
     experiments, _ = callback.get_results()
     design = case
     design['policy'] = policy['name']
     design['model'] = name
     
     names = rf.get_names(experiments.dtype)
     for name in names:
         self.assertEqual(experiments[name][0], design[name])

    def _rotate_subset(self, value, orig_experiments, logical): 
        '''
        rotate a subset
        
        Parameters
        ----------
        value : list of strings
        orig_experiment : numpy structured array
        logical : boolean array
        
        '''
        list_dtypes = [(name, "<f8") for name in value]
        
        #cast everything to float
        drop_names = set(rf.get_names(orig_experiments.dtype)) - set(value)
        orig_subset = rf.drop_fields(orig_experiments, drop_names, 
                                               asrecarray=True)
        subset_experiments = orig_subset.astype(list_dtypes).view('<f8').reshape(orig_experiments.shape[0], len(value))
 
        #normalize the data
        mean = np.mean(subset_experiments,axis=0)
        std = np.std(subset_experiments, axis=0)
        std[std==0] = 1 #in order to avoid a devision by zero
        subset_experiments = (subset_experiments - mean)/std
        
        #get the experiments of interest
        experiments_of_interest = subset_experiments[logical]
        
        #determine the rotation
        rotation_matrix =  self._determine_rotation(experiments_of_interest)
        
        #apply the rotation
        subset_experiments = np.dot(subset_experiments,rotation_matrix)
        return rotation_matrix, subset_experiments

 def test_prim_init_select(self):
     self.results = util.load_flu_data()
     self.classify = flu_classify        
     
     experiments, outcomes = self.results
     
     unc = recfunctions.get_names(experiments.dtype)
     
     # test initialization, including t_coi calculation in case of searching
     # for results equal to or higher than the threshold
     outcomes['death toll'] = outcomes['deceased population region 1'][:, -1]
     results = experiments, outcomes
     threshold = 10000
     prim_obj = prim.setup_prim(results, classify='death toll', 
                          threshold_type=prim.ABOVE, threshold=threshold,
                          incl_unc=unc)
     
     value = np.ones((experiments.shape[0],))
     value = value[outcomes['death toll'] >= threshold].shape[0]
     self.assertTrue(prim_obj.t_coi==value)
             
     # test initialization, including t_coi calculation in case of searching
     # for results equal to or lower  than the threshold
     threshold = 1000
     prim_obj = prim.setup_prim(results, classify='death toll', 
                          threshold_type=prim.BELOW, 
                          threshold=threshold)
     
     value = np.ones((experiments.shape[0],))
     value = value[outcomes['death toll'] <= threshold].shape[0]
     self.assertTrue(prim_obj.t_coi==value)
     
     prim.setup_prim(self.results, self.classify, threshold=prim.ABOVE)

def plot_cdfs(x, y, ccdf=False):
    '''plot cumulative density functions for each column in x, based on the 
    classification specified in y.
    
    Parameters
    ----------
    x : recarray
        the experiments to use in the cdfs
    y : ndaray 
        tthe categorization for the data
    ccdf : bool, optional
           if true, plot a complementary cdf 
           instead of a normal cdf.
    
    '''
    
    uncs = rf.get_names(x.dtype)
    cp = sns.color_palette()
    
    n_col = 4
    n_row = len(uncs)//n_col +1
    size = 3 
    aspect = 1
    figsize = n_col * size * aspect, n_row * size
    fig, axes = plt.subplots(n_row, n_col,
                             figsize=figsize,
                             squeeze=False)

    for i, unc in enumerate(uncs):
        discrete = False
        
        i_col = i % n_col
        i_row = i // n_col
        ax = axes[i_row, i_col]
        
        data = x[unc]
        if x.dtype[unc] == np.dtype('O'):
            discrete = True
        plot_cdf(ax, unc, data, y, discrete, ccdf=ccdf)
    
    # last row might contain empty axis, 
    # let's make them disappear
    i_row = len(uncs) // n_col
    i_col = len(uncs) % n_col
    for i_col in range(i_col, n_col):
        ax = axes[i_row, i_col]
        ax.set_xticklabels([])
        ax.set_xticks([])
        ax.set_yticklabels([])
        ax.set_yticks([])
        
        sns.despine(ax=ax, top=True, right=True, left=True, bottom=True)
    
    proxies, labels = build_legend(x, y)
    
    fig.legend(proxies, labels, "upper center")

    return fig

def _in_box(x, boxlim):
    '''
     
    returns the indices of the data points that are within the 
    box_lims.
    
    Parameters
    ----------
    x : numpy structured array
    boxlim : numpy structured array
    
    
    Returns
    -------
    ndarray
        valid numpy indices on x
    
    '''
    logical = np.ones(x.shape[0], dtype=np.bool)
    
    dims = recfunctions.get_names(boxlim.dtype)

    for name in dims:
        value = x.dtype.fields.get(name)[0]
        
        if value == 'object':
            entries = boxlim[name][0]
            l = np.ones( (x.shape[0], len(entries)), dtype=np.bool)
            for i,entry in enumerate(entries):
                if type(list(entries)[0]) not in (str, float, int):
                    bools = []                
                    for element in list(x[name]):
                        if element == entry:
                            bools.append(True)
                        else:
                            bools.append(False)
                    l[:, i] = np.asarray(bools, dtype=bool)
                else:
                    l[:, i] = x[name] == entry
            l = np.any(l, axis=1)
            logical = logical & l
        else:
            logical = logical & (boxlim[name][0] <= x[name] )&\
                                        (x[name] <= boxlim[name][1])                
    
    indices = np.where(logical==True)
    
    assert len(indices)==1
    indices = indices[0]
    
    return indices

 def test_init(self):
     # let's add some uncertainties to this
     uncs = [ParameterUncertainty((0,1), "a"),
            ParameterUncertainty((0,1), "b")]
     outcomes = [Outcome("test", time=True)]
     callback = DefaultCallback(uncs, outcomes, nr_experiments=100)
     
     self.assertEqual(callback.i, 0)
     self.assertEqual(callback.nr_experiments, 100)
     self.assertEqual(callback.cases.shape[0], 100)
     self.assertEqual(callback.outcomes, outcomes)
     
     names = rf.get_names(callback.cases.dtype)
     names = set(names)
     self.assertEqual(names, {'a', 'b', 'policy', 'model'})
     self.assertEqual(callback.results, {})

    def test_drop_restriction(self):
        x = np.array([(0,1,2),
                      (2,5,6),
                      (3,2,1)], 
                     dtype=[('a', np.float),
                            ('b', np.float),
                            ('c', np.float)])
        y = np.array([1,1,0])
         
        prim_obj = Prim(x, y, threshold=0.8)
        box = PrimBox(prim_obj, prim_obj._box_init, prim_obj.yi)
 
        new_box_lim = np.array([(0,1,1),
                                (2,2,6)], 
                                dtype=[('a', np.float),
                                       ('b', np.float),
                                       ('c', np.float)])
        indices = np.array([0,1], dtype=np.int)
        box.update(new_box_lim, indices)
         
        box.drop_restriction('b')
         
        correct_box_lims = np.array([(0,1,1),
                                     (2,5,6)], 
                                    dtype=[('a', np.float),
                                           ('b', np.float),
                                           ('c', np.float)]) 
               
        box_lims = box._box_lims[-1]
        names = recfunctions.get_names(correct_box_lims.dtype)
        
        for entry in names:
            lim_correct = correct_box_lims[entry]
            lim_box = box_lims[entry]
            for i in range(len(lim_correct)):
                self.assertEqual(lim_correct[i], lim_box[i])
         
        self.assertEqual(box.peeling_trajectory['mean'][2], 1)
        self.assertEqual(box.peeling_trajectory['coverage'][2], 1)
        self.assertEqual(box.peeling_trajectory['density'][2], 1)
        self.assertEqual(box.peeling_trajectory['res dim'][2], 1)
        self.assertEqual(box.peeling_trajectory['mass'][2], 2/3)

def _determine_restricted_dims(box_lims, box_init):
    '''
    
    determine which dimensions of the given box are restricted compared 
    to compared to the initial box that contains all the data
    
    Parameters
    ----------
    box_lims : structured numpy array
               a specific box limit
    box_init : structured numpy array
               the initial box containing all data points
    
    '''

    logical = _compare(box_init, box_lims)
    u = np.asarray(recfunctions.get_names(box_lims.dtype), 
                   dtype=object)
    dims = u[logical==False]
    return dims

def setup_prim(results, classify, incl_unc=[], **kwargs):
    """Helper function for setting up the prim algorithm
    
    Parameters
    ----------
    results : tuple of structured array and dict with numpy arrays
              the return from :meth:`perform_experiments`.
    classify : string, function or callable
               either a string denoting the outcome of interest to 
               use or a function. 
    kwargs : valid keyword arguments for prim.Prim
    
    Returns
    -------
    a Prim instance
    
    Raises
    ------
    PrimException 
        if data resulting from classify is not a 1-d array. 
    TypeError 
        if classify is not a string or a callable.
    
    """
    
    if not incl_unc:
        x = np.ma.array(results[0])
    else:
        drop_names = set(rf.get_names(results[0].dtype))-set(incl_unc)
        x = rf.drop_fields(results[0], drop_names, asrecarray = True)
    if type(classify)==StringType:
        y = results[1][classify]
    elif callable(classify):
        y = classify(results[1])
    else:
        raise TypeError("unknown type for classify")
    
    return Prim(x,y, **kwargs)

def make_box(x):
    '''
    Make a box that encompasses all the data
    
    Parameters
    ----------
    x : structured numpy array
    '''
    # get the types in the order they appear in the numpy array
    types = [(v[1], k, v[0].name) for k, v in six.iteritems(x.dtype.fields)]
    types = sorted(types)
    
    # convert any bool types to object to store set(False, True)
    ntypes = [(k, 'object' if t == 'bool' else t) for (_, k, t) in types]
    
    # create box limits
    box = np.zeros((2, ), ntypes)
    names = recfunctions.get_names(x.dtype)
    
    for name in names:
        dtype = box.dtype.fields.get(name)[0]
        values = x[name]
        
        if isinstance(values, np.ma.MaskedArray):
            values = values.compressed()
        
        if dtype == 'object':
            try:
                values = set(values)
                box[name][:] = values
            except TypeError as e:
                logging.getLogger(__name__).warning("{} has unhashable values".format(name))
                raise e
        else:
            box[name][0] = np.min(values, axis=0)
            box[name][1] = np.max(values, axis=0)
               
    return box  

def setup_cart(results, classify, incl_unc=[], mass_min=0.05):
    """helper function for performing cart in combination with data
    generated by the workbench. 

    Parameters
    ----------
    results : tuple of structured array and dict with numpy arrays
              the return from :meth:`perform_experiments`.
    classify : string, function or callable
               either a string denoting the outcome of interest to 
               use or a function. 
    incl_unc : list of strings
    mass_min : float


    Raises
    ------
    TypeError 
        if classify is not a string or a callable.

    """

    if not incl_unc:
        x = np.ma.array(results[0])
    else:
        drop_names = set(recfunctions.get_names(
            results[0].dtype))-set(incl_unc)
        x = recfunctions.drop_fields(results[0], drop_names, asrecarray=True)
    if isinstance(classify, six.string_types):
        y = results[1][classify]
        mode = sdutil.REGRESSION
    elif callable(classify):
        y = classify(results[1])
        mode = sdutil.BINARY
    else:
        raise TypeError("unknown type for classify")

    return CART(x, y, mass_min, mode=mode)

    def __init__(self, 
                 x,
                 y, 
                 threshold = None, 
                 threshold_type = ">",
                 obj_func = lenient1, 
                 peel_alpha = 0.05, 
                 paste_alpha = 0.05,
                 mass_min = 0.05, 
                 include = None,
                 exclude = None,
                 coi = None):
        """Creates a new PRIM object.
        
        The PRIM object maintains the current state of the PRIM algorithm,
        recording the PRIM boxes found thus far, the remaining (uncaptured)
        cases of interest in the dataset, and provides methods for finding the
        next PRIM box and viewing statistics.
        
        Parameters
        ----------
        x : a matrix-like object (pandas.DataFrame, numpy.recarray, etc.)
            the independent variables
        y : a list-like object, the column name (str), or callable
            the dependent variable either provided as a list-like object
            classifying the data into cases of interest (e.g., False/True),
            a list-like object storing the raw variable value (in which case
            a threshold must be given), a string identifying the dependent
            variable in x, or a function called on each row of x to compute the
            dependent variable
        threshold : float
            threshold for identifying cases of interest
        threshold_type : str
            comparison operator used whwen identifying cases of interest
        obj_func : callable (default: lenient1)
            a function that computes the objective function (peeling criteria)
        peel_alpha : float (default: 0.05) 
            parameter controlling the peeling stage
        paste_alpha : float (default: 0.05)
            parameter controlling the pasting stage
        mass_min : float (default: 0.05)
            minimum mass of a box
        include : list of str
            the names of variables included in the PRIM analysis
        exclude : list of str
            the names of variables excluded from the PRIM analysis
        coi : str or list of str
            if y contains strings, coi identifies which string is the case of
            interest
        """
        
        # Ensure the input x is a numpy matrix/array
        if isinstance(x, pd.DataFrame):
            x = x.to_records(index=False)
        elif isinstance(x, np.ma.MaskedArray):
            pass
        else:
            x = pd.DataFrame(x).to_records(index=False)
            
        # if y is a string or function, compute the actual response value
        # otherwise, ensure y is a numpy matrix/array
        if isinstance(y, six.string_types):
            key = y
            y = x[key]
            
            if exclude:
                exclude = list(exclude) + [key]
            else:
                exclude = [key]
        elif six.callable(y):
            fun = y
            y = np.apply_along_axis(fun, 0, x)
        elif isinstance(y, pd.DataFrame) or isinstance(y, pd.Series):
            y = y.values
        elif isinstance(y, np.ma.MaskedArray):
            pass
        else:
            y = np.asarray(y)
            
        # convert include/exclude arguments to lists if they are strings
        if include and isinstance(include, six.string_types):
            include = [include]
            
        if exclude and isinstance(exclude, six.string_types):
            exclude = [exclude]     
            
        # include or exclude columns from the analysis
        if include:
            if isinstance(include, six.string_types):
                include = [include]

            drop_names = set(rf.get_names(x.dtype))-set(include)
            x = rf.drop_fields(x, drop_names, asrecarray=True)
        
        if exclude:
            if isinstance(exclude, six.string_types):
                exclude = [exclude]

            drop_names = set(exclude) 
            x = rf.drop_fields(x, drop_names, asrecarray=True)
#.........这里部分代码省略.........

    def __init__(self,
                 x,
                 y, 
                 threshold = None, 
                 threshold_type = ">",
                 include = None,
                 exclude = None,
                 **kwargs):
        """Generates a decision tree for classification.
        
        Parameters
        ----------
        x : a matrix-like object (pandas.DataFrame, numpy.recarray, etc.)
            the independent variables
        y : a list-like object, the column name (str), or callable
            the dependent variable either provided as a list-like object
            classifying the data into cases of interest (e.g., False/True),
            a list-like object storing the raw variable value (in which case
            a threshold must be given), a string identifying the dependent
            variable in x, or a function called on each row of x to compute the
            dependent variable
        threshold : float
            threshold for identifying cases of interest
        threshold_type : str
            comparison operator used when identifying cases of interest
        include : list of str
            the names of variables included in the PRIM analysis
        exclude : list of str
            the names of variables excluded from the PRIM analysis
        """
        super(Cart, self).__init__()
        
        # Ensure the input x is a numpy matrix/array
        if isinstance(x, pd.DataFrame):
            x = x.to_records(index=False)
        elif isinstance(x, np.ma.MaskedArray):
            pass
        else:
            x = pd.DataFrame(x).to_records(index=False)
            
        # if y is a string or function, compute the actual response value
        # otherwise, ensure y is a numpy matrix/array
        if isinstance(y, six.string_types):
            key = y
            y = x[key]
            
            if exclude:
                exclude = list(exclude) + [key]
            else:
                exclude = [key]
        elif six.callable(y):
            fun = y
            y = np.apply_along_axis(fun, 0, x)
        elif isinstance(y, pd.DataFrame) or isinstance(y, pd.Series):
            y = y.values
        elif isinstance(y, np.ma.MaskedArray):
            pass
        else:
            y = np.asarray(y)
            
        # convert include/exclude arguments to lists if they are strings
        if include and isinstance(include, six.string_types):
            include = [include]
            
        if exclude and isinstance(exclude, six.string_types):
            exclude = [exclude]     
            
        # include or exclude columns from the analysis
        if include:
            if isinstance(include, six.string_types):
                include = [include]

            drop_names = set(rf.get_names(x.dtype))-set(include)
            x = rf.drop_fields(x, drop_names, asrecarray=True)
        
        if exclude:
            if isinstance(exclude, six.string_types):
                exclude = [exclude]

            drop_names = set(exclude) 
            x = rf.drop_fields(x, drop_names, asrecarray=True)
            
        # apply the threshold if 
        if threshold:
            if six.callable(threshold):
                y = np.apply_along_axis(threshold, 0, y)
            else:
                # The syntax for threshold_type is "x <op> <threshold>", e.g.,
                # "x > 0.5".  However, partial only supports positional
                # arguments for built-in operators.  Thus, we must assign the
                # threshold to the first position and use a different operator.
                # For example, "x > 0.5" must be evaluated as "0.5 < x".
                OPERATORS = {"<" : operator.ge,
                             ">" : operator.le,
                             "<=" : operator.gt,
                             ">=" : operator.lt,
                             "=" : operator.eq}
                
                op = OPERATORS[threshold_type]
                y = np.apply_along_axis(functools.partial(op, threshold), 0, y)
#.........这里部分代码省略.........

def get_ex_feature_scores(x, y, mode=CLASSIFICATION, nr_trees=250, 
                          max_features='auto', max_depth=None, 
                          min_samples_split=2, min_samples_leaf=1, 
                          min_weight_fraction_leaf=0, max_leaf_nodes=None,
                          bootstrap=True, oob_score=True, random_state=None): 
    '''
    Get feature scores using extra trees

    Parameters
    ----------
    x : structured array
    y : 1D nd.array
    mode : {CLASSIFICATION, REGRESSION}
    nr_trees : int, optional
               nr. of trees in forest (default=250)
    max_features : int, optional
                   see http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesClassifier.html
    max_depth : int, optional 
                see http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesClassifier.html
    min_samples_split : int, optional
                  see http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesClassifier.html
    min_samples_leaf : int, optional
                       see http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesClassifier.html
    min_weight_fraction_leaf : float, optional
                               see http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesClassifier.html
    max_leaf_nodes: int or None, optional
                    see http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesClassifier.html
    bootstrap : bool, optional
                see http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesClassifier.html
    oob_score : bool, optional
                see http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesClassifier.html
    random_state : int, optional
                   see http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesClassifier.html
    
    Returns
    -------
    pandas DataFrame
        sorted in descending order of tuples with uncertainty and feature 
        scores 
    object
        either ExtraTreesClassifier or ExtraTreesRegressor
    
    '''
    
    uncs = recfunctions.get_names(x.dtype)
    x = _prepare_experiments(x)
    
    if mode==CLASSIFICATION:
        etc = ExtraTreesClassifier
        criterion='gini'
    elif mode==REGRESSION:
        etc = ExtraTreesRegressor
        criterion = 'mse'
    else:
        raise ValueError('{} not valid for mode'.format(mode))
    
    extra_trees = etc(n_estimators=nr_trees, 
                      criterion=criterion, 
                      max_features=max_features, 
                      max_depth=max_depth,
                      min_samples_split=min_samples_split,
                      min_samples_leaf=min_samples_leaf,
                      min_weight_fraction_leaf=min_weight_fraction_leaf,
                      max_leaf_nodes=max_leaf_nodes,
                      bootstrap=bootstrap,
                      oob_score=oob_score,
                      random_state=random_state)
    extra_trees.fit(x,y)

    importances = extra_trees.feature_importances_

    importances = zip(uncs, importances)
    importances = list(importances)
    importances.sort(key=itemgetter(1), reverse=True)

    importances = pd.DataFrame(importances)

    return importances, extra_trees

 def test_store_cases(self):
     nr_experiments = 3
     uncs = [RealParameter("a", 0, 1),
             RealParameter("b", 0, 1),
             CategoricalParameter('c', [0, 1, 2]),
             IntegerParameter("d", 0, 1)]
     outcomes = [TimeSeriesOutcome("test")]
     constraints = []
     case = {unc.name:random.random() for unc in uncs}
     case["c"] = int(round(case["c"]*2))
     case["d"] = int(round(case["d"]))
     
     model = NamedObject('test')
     policy  = Policy('policy')
     scenario = Scenario(**case)
     experiment = Case(0, model.name, policy, scenario, 0)
  
     callback = DefaultCallback(uncs, [],outcomes, constraints,
                                nr_experiments=nr_experiments,
                                reporting_interval=1)
     model_outcomes = {outcomes[0].name: 1}
     model_constraints = {}
     callback(experiment, model_outcomes, model_constraints)
      
     experiments, _ = callback.get_results()
     design = case
     design['policy'] = policy.name
     design['model'] = model.name
     design['scenario_id'] = scenario.name
     
     names = rf.get_names(experiments.dtype)
     for name in names:
         entry_a = experiments[name][0]
         entry_b = design[name]
         
         self.assertEqual(entry_a, entry_b, "failed for "+name)
          
     # with levers
     nr_experiments = 3
     uncs = [RealParameter("a", 0, 1),
             RealParameter("b", 0, 1)]
     levers = [RealParameter("c", 0, 1),
               RealParameter("d", 0, 1)]
     outcomes = [TimeSeriesOutcome("test")]
     case = {unc.name:random.random() for unc in uncs}
     
     model = NamedObject('test')
     policy  = Policy('policy', c=1, d=1)
     scenario = Scenario(**case)
     experiment = Case(0, model.name, policy, scenario, 0)
  
     callback = DefaultCallback(uncs, levers,outcomes,constraints, 
                                nr_experiments=nr_experiments,
                                reporting_interval=1)
     model_outcomes = {outcomes[0].name: 1}
     model_constraints = {}
     callback(experiment, model_outcomes, model_constraints)
      
     experiments, _ = callback.get_results()
     design = case
     design['c'] = 1
     design['d'] = 1
     design['policy'] = policy.name
     design['model'] = model.name
     design['scenario_id'] = scenario.name
     
     names = rf.get_names(experiments.dtype)
     
     for name in names:
         self.assertEqual(experiments[name][0], design[name])