def _create_test_data(self):
     X, y = datasets.make_friedman1(n_samples=20, random_state=13)
     X = pd.DataFrame(X)
     Y = Response.from_array(y / y.max())
     Z = Partition(size=X.shape[0], folds=5, reps=1, total_size=X.shape[0])
     Z.set(max_reps=1, max_folds=0)
     return Container(X), Y, Z

 def create_data(self):
     X = copy.deepcopy(self.ds)
     Y = Response.from_array(X.pop('Claim_Amount').values)
     X = X.take(range(21,29),axis=1)
     Z = Partition(size=X.shape[0],total_size=X.shape[0]+20,folds=5,reps=5)
     Z.set(max_reps=1,max_folds=0)
     return Container(X),Y,Z

    def test_ngrams_words_calculates_ace(self):
        xdata = np.repeat(np.array(['dog cat dog cat', 'cat dog dog cat', 'cat dog cat dog', 'dog dog cat cat']), 25).reshape(-1, 1)
        perm = np.random.permutation(xdata.shape[0])
        X = Container()
        X.add(xdata[perm, :])
        y = np.repeat(np.array([1, 1, 0, 0]), 25)[perm]
        Z = Partition(size=xdata.shape[0], reps=5)
        Z.set(max_folds=0, max_reps=2)
        taskbow = AutoTunedWordGramClassifier('num=1;ma=LogLoss')
        taskbow.fit(X, y, Z)
        predictions = taskbow.predict(X, y, Z)
        report = taskbow.report()
        for p in Z:
            key = (p['r'], p['k'])
            self.assertTrue('var_imp_info' in report[key])
            self.assertTrue(report[key]['var_imp_info'] < 0.1)

        taskw = AutoTunedWordGramClassifier('num=4;ma=LogLoss')
        taskw.fit(X, y, Z)
        transform = taskw.transform(X, y, Z)
        predictions = taskw.predict(X, y, Z)
        report = taskw.report()
        for p in Z:
            key = (p['r'], p['k'])
            print key
            print report[key]['var_imp_info']
            print predictions(**p).ravel()
            print predictions(**p) == y
            self.assertTrue('var_imp_info' in report[key])
            self.assertGreater(report[key]['var_imp_info'],  0.9)

    def test_clf_early_stop_gridsearch_weights(self, mocklogloss):
        """Test clf passes weights to the loss function if early-stopping is in effect when doing gridsearch. """
        def weight_loss(actual, pred, weights):
            print "Test"
            if np.all(weights[actual == 1] == 10.) and \
               np.all(weights[actual == -1] == 1.):
                raise ValueError("Weights passed successfully")
            else:
                assert(False)
                return np.sum(pred) - 50.0
        mocklogloss.method = weight_loss
        x, Y = make_hastie_10_2(n_samples=300, random_state=41)
        X = Container()
        X.add(x)
        Z = Partition(X.shape[0], max_reps=2, max_folds=0)
        Z.set(max_reps=1, max_folds=1)
        wt = {'weight': pd.Series(2.0 + 9.0 * (Y == 1).astype(float))}

        # Add weights to container
        X.initialize(wt)
        task = ESGBC('s=1;n=10;md=[2];ls=1;lr=[0.1, 0.000001];t_m=Weighted LogLoss')

        task.fit(X, Y, Z)
        # Assert the patched loss function was passed the weights
        self.assertTrue(mocklogloss.called)
        # The third argument is weight, we should be passed two values
        passed_weights = mocklogloss.call_args[0][2]
        passed_actuals = mocklogloss.call_args[0][0]
        self.assertEqual(len(np.unique(passed_weights)), 2)
        print passed_weights
        self.assertTrue(np.all(passed_weights[passed_actuals == -1] == 2))
        self.assertTrue(np.all(passed_weights[passed_actuals == 1] == 11))

 def create_reg_count_syn_data(self, reps=1):
     X, y = syn_counts(n_samples=500, random_state=13)
     X = pd.DataFrame(data=X, columns=map(unicode, range(X.shape[1])))
     Y = Response.from_array(y)
     Z = Partition(size=X.shape[0], folds=5, reps=reps,total_size=X.shape[0])
     Z.set(max_reps=reps,max_folds=0)
     X = Container(dataframe=X)
     return X, Y, Z

 def create_bin_large_data(self,reps=1):
     X = copy.deepcopy(self.ds3)
     Y = X.pop('SeriousDlqin2yrs').values
     Y = Response.from_array(Y)
     Z = Partition(size=X.shape[0],folds=5,reps=reps,total_size=X.shape[0])
     Z.set(max_reps=reps,max_folds=0)
     X = Container(dataframe=X)
     return X,Y,Z

 def generate_data(self,nrows=5000,ncols=4,seed=56):
     colnames = [str(i) for i in xrange(ncols)]
     np.random.seed(seed)
     x = np.random.randn(nrows, ncols)
     X = Container()
     X.add(x, colnames=colnames)
     Y=x[:,0]+x[:,1]**2+x[:,2]*x[:,1]
     Z = Partition(size=nrows,folds=1,reps=1,total_size=nrows)
     Z.set(max_reps=1,max_folds=0)
     return X,Y,Z

 def create_bin_data(self,reps=1, rows=None):
     X = copy.deepcopy(self.ds)
     if rows is not None and rows < X.shape[0]:
         X = X[:rows]
     Y = X.pop('SeriousDlqin2yrs').values
     Y = Response.from_array(Y)
     Z = Partition(size=X.shape[0],folds=5,reps=reps,total_size=X.shape[0])
     Z.set(max_reps=reps,max_folds=0)
     X = Container(dataframe=X)
     return X,Y,Z

    def test_max_reps0(self):
        part = Partition(size=100,folds=5,reps=5,total_size=120)
        part.set( max_reps=0)
        answer = """...++....+.....+........+.....+++............+.+.+..+..+.+..+....+..........+...+....+...+..........
++....+................+..+..+....+..+..++.+......+.....+....++...................++.......++..+....
..+........+..+.+....+......+......+......+...+............+...+..+......+....++.+.....+..+.....+.+.
.....+.+..+.++....+++.+..+.......+..+.+.........+.....+...+...............+...........+.+....+......
........+........+.........+...........+....+......+.+..........+..++++++..+.+......+.........+..+.+
"""
        self.assertEqual(plot_partition(part,100),answer)

 def generate_data(self, nrows=5000, ncols=4, seed=56):
     colnames = ['X'+str(i) for i in xrange(ncols)]
     np.random.seed(seed)
     x = abs(np.random.randn(nrows, ncols))
     x[:, 1] = x[:, 0]*1.5 + x[:, 1]
     X = Container()
     X.initialize({'weight': pd.Series(np.ones(nrows))})
     X.add(x, colnames=colnames)
     Z = Partition(size=nrows, folds=1, reps=1, total_size=nrows)
     Z.set(max_reps=1, max_folds=0)
     Y = 3 * (x[:, 1] - x[:, 0]) + 0.2 * x[:, 3]
     return X, Y, Z

    def test_no_gcv(self):
        part = Partition(size=100,folds=5,reps=5,total_size=120)
        part.set( max_reps=0)
        part.set( no_test=True)
        part.set( no_gcv=True )
        answer = """|.|.|...|.|.|.........|......|....||....|...|.......|.||...|....|......|.......||...................
...|..........||..||......|.|..|.....|...........|.|.|.......|.|..||..............|...|......||.....
.|...........|.......|........|.......|..|......|.......||..|....|..||...........|..|..||.||.....|..
.....||....|........|...........||..|.....||.|||..|...........|.......|....|.|.......|..........|..|
.......|.|......||.....|||.|...........|..................|.............|||.|.|....|.....|..|..|..|.
"""
        self.assertEqual(plot_partition(part,100),answer)

 def test_p100f1r1t3(self):
     part = Partition(100, folds=1, reps=1, testfrac=3,total_size=120)
     self.assertEqual(len(part),1)
     for i in part:
         self.assertTupleEqual((len(part.T(**i)),len(part.V(**i)),len(part.S(**i))),(0,66,34))
     answer = "++++||++|+++|++|+||+++|+||||++|+|+|||+|+++|++|++++++|+++++++++||+++|+++++++|++|+++|+|++|++|||+||++++\n"
     self.assertEqual(plot_partition(part,100),answer)
     part.set(samplepct=50)
     for i in part:
         self.assertTupleEqual((len(part.T(**i)),len(part.V(**i)),len(part.S(**i))),(0, 40, 20))
     answer = "+ ++  + | +  ++|+||++ | |||       |||+  + |++|++++ +|+++++++++ |  + + +++   +   ++ +| +|   || |   ++\n"
     self.assertEqual(plot_partition(part,100),answer)

 def test_rrfc_auto(self):
     """Regression test for mtry = auto (should produce
     ncols / 3
     """
     x = np.random.randn(84).reshape((-1, 12))
     y = np.random.randint(0, 2, size=x.shape[1])
     X = Container()
     X.add(x)
     r_rf = RRFBMC()
     Z = Partition(x.shape[0], folds=5, reps=1,total_size=x.shape[0])
     Z.set(max_reps=1, max_folds=0)
     r_rf._modify_parameters(X(), y)
     self.assertEqual(r_rf.parameters['mtry'],  4)

    def test_transform_smoketest_no_pruning(self):
        X = np.random.randn(100, 10)
        Y = np.random.randn(100, 1)
        XC = Container()
        XC.add(X)
        Z = Partition(size=100, reps=5, folds=5)
        Z.set(max_reps=1, max_folds=1)

        pe = PyEarthTransformer('dp=0')
        pe.fit(XC, Y, Z)

        out = pe.transform(XC, Y, Z)
        for p in Z:
            self.assertEqual(out(**p).shape[0], 100)

 def test_fit_transform_smoke(self):
     x = self.create_dataframe(100, 'NNCUI')
     cf = CFConverter()
     Y = Response.from_array(np.random.rand(100))
     Z = Partition(100, seed=1)
     Z.set(partitions=[(0, -1)])
     cf.fit(Container(x), Y, Z)
     out = cf.transform(Container(x), Y, Z)
     for p in Z:
         ctx = out.get_user_item_context(**p)
         self.assertEqual(ctx.user_id, 0)
         self.assertEqual(ctx.item_id, 1)
         self.assertEqual(ctx.n_users, 10)
         self.assertEqual(ctx.n_items, 10)

 def create_reg_data(self,reps=1, rows=None, categoricals=False):
     X = copy.deepcopy(self.ds2)
     if rows is not None and rows < X.shape[0]:
         X = X[:rows]
     Y = X.pop('Claim_Amount').values
     if not categoricals:
         X = X.take(range(21,29),axis=1)
     else:
         X = X.take(range(5,29),axis=1)
     Y = Response.from_array(Y)
     Z = Partition(size=X.shape[0],folds=5,reps=reps,total_size=X.shape[0])
     Z.set(max_reps=reps,max_folds=0)
     X = Container(dataframe=X)
     return X,Y,Z

 def test_XTransform_sev_fit_sets_mask(self):
     """Test that severity transform runs successfully and
     that after Vertex.fit() it contains the correct mask
     """
     xdata = np.random.randn(400).reshape((40, -1))
     ydata = Response.from_array(np.random.randint(0, 2, size=40).reshape(-1, 1))
     xcont = Container()
     xcont.add(xdata)
     Z = Partition(40, total_size=24,folds=5, reps=1)
     Z.set(max_reps=1, max_folds=5)
     vertex = Vertex(['RFR sev'], 'id')
     vertex.fit(xcont, ydata, Z)
     for p in xcont:
         self.assertEqual(xcont.get_mask(**p), ydata > 0)

 def test_multinomial_reg_predictions_median_in_range_unique_values(self):
     """Test that large number of values raise an error. """
     X, Y, Z = self.create_reg_data()
     Y = np.round(np.log(Y)) + 5  # Greater than 1 to ensure bounds don't work for probabilities
     # need to have fewer than ten values
     task = self.check_task('MNGBR n=10;cm=median;md=[3, 4];t_n=1;t_f=0.15', X, Y, Z, transform=True, standardize=False)
     est = task.model[(0, -1)]
     Z = Partition(size=X.dataframe.shape[0], folds=5, reps=5, total_size=X.dataframe.shape[0])
     Z.set(max_reps=1, max_folds=0)
     C = Container()
     C.add(X.dataframe.values)
     # task.model[(-1, -1)] = task.model[(0, -1)]
     preds = task.predict(C, Y, Z)(r=0, k=-1)
     assert(np.unique(Y).max() >= preds.max())
     assert(np.unique(Y).min() <= preds.min())

 def test_ngrams_smoketest_regressor(self):
     xdata = np.repeat(np.array(['dog cat dog cat', 'cat dog dog cat', 'cat dog cat dog', 'dog dog cat cat']), 25).reshape(-1, 1)
     X = Container()
     X.add(xdata)
     y = np.repeat(np.array([3, 2, 0, 0]), 25)
     Z = Partition(size=xdata.shape[0], reps=5)
     Z.set(max_folds=0, max_reps=2)
     taskw = AutoTunedWordGramRegressor('num=[1, 2, 3]')
     taskw.fit(X, y, Z)
     transform = taskw.transform(X, y, Z)
     predictions = taskw.predict(X, y, Z)
     taskc = AutoTunedCharGramRegressor('num=[2, 3, 4]')
     taskc.fit(X, y, Z)
     transform = taskc.transform(X, y, Z)
     predictions = taskc.predict(X, y, Z)

 def test_SparseStandardize_coo(self):
     """Test that it returns sparse for sparse data in COO format
     """
     x = sp.rand(500, 500, format='coo')
     X = Container()
     X.add(x)
     self.assertTrue(sp.issparse(X()))
     y = np.random.randint(0, 1, 50)
     Z = Partition(size=x.shape[0], folds=5, reps=5,total_size=x.shape[0])
     Z.set(max_reps=5, max_folds=0)
     model = Standardize()
     res = model.fit(X, y, Z)
     out = res.transform(X, y, Z)
     self.assertTrue(sp.issparse(out()))
     self.assertTrue(sp.issparse(out(r=1, k=0)))