def setupClass(cls):
     cls.endog = endog = pandas.DataFrame(np.random.random((10,4)),
                                  columns=['y_1', 'y_2', 'y_3', 'y_4'])
     exog =  pandas.DataFrame(np.random.random((10,2)),
                              columns=['x_1','x_2'])
     exog.insert(0, 'const', 1)
     cls.exog = exog
     cls.data = sm_data.handle_data(cls.endog, cls.exog)
     nrows = 10
     nvars = 3
     neqs = 4
     cls.col_input = np.random.random(nvars)
     cls.col_result = pandas.Series(cls.col_input,
                                       index=exog.columns)
     cls.row_input = np.random.random(nrows)
     cls.row_result = pandas.Series(cls.row_input,
                                       index=exog.index)
     cls.cov_input = np.random.random((nvars, nvars))
     cls.cov_result = pandas.DataFrame(cls.cov_input,
                                        index = exog.columns,
                                        columns = exog.columns)
     cls.cov_eq_input = np.random.random((neqs, neqs))
     cls.cov_eq_result = pandas.DataFrame(cls.cov_eq_input,
                                           index=endog.columns,
                                           columns=endog.columns)
     cls.col_eq_input = np.random.random((nvars, neqs))
     cls.col_eq_result = pandas.DataFrame(cls.col_eq_input,
                                           index=exog.columns,
                                           columns=endog.columns)
     cls.xnames = ['const', 'x_1', 'x_2']
     cls.ynames = ['y_1', 'y_2', 'y_3', 'y_4']
     cls.row_labels = cls.exog.index

 def test_extra_kwargs_2d(self):
     sigma = np.random.random((25, 25))
     sigma = sigma + sigma.T - np.diag(np.diag(sigma))
     data = sm_data.handle_data(self.y, self.X, 'drop', sigma=sigma)
     idx = ~np.isnan(np.c_[self.y, self.X]).any(axis=1)
     sigma = sigma[idx][:,idx]
     np.testing.assert_array_equal(data.sigma, sigma)

 def setupClass(cls):
     super(TestStructarrays, cls).setupClass()
     cls.endog = np.random.random(9).view([("y_1", "f8")]).view(np.recarray)
     exog = np.random.random(9 * 3).view([("const", "f8"), ("x_1", "f8"), ("x_2", "f8")]).view(np.recarray)
     exog["const"] = 1
     cls.exog = exog
     cls.data = sm_data.handle_data(cls.endog, cls.exog)
     cls.xnames = ["const", "x_1", "x_2"]
     cls.ynames = "y_1"

 def test_drop(self):
     y = self.y
     X = self.X
     combined = np.c_[y, X]
     idx = ~np.isnan(combined).any(axis=1)
     y = y[idx]
     X = X[idx]
     data = sm_data.handle_data(self.y, self.X, 'drop')
     np.testing.assert_array_equal(data.endog, y)
     np.testing.assert_array_equal(data.exog, X)

 def setup_class(cls):
     cls.endog = pandas.DataFrame(np.random.random(10), columns=['y_1'])
     mi = pandas.MultiIndex.from_product([['x'], ['1', '2']])
     exog = pandas.DataFrame(np.random.random((10, 2)), columns=mi)
     exog_flattened_idx = pandas.Index(['const', 'x_1', 'x_2'])
     exog.insert(0, 'const', 1)
     cls.exog = exog
     cls.data = sm_data.handle_data(cls.endog, cls.exog)
     nrows = 10
     nvars = 3
     cls.col_input = np.random.random(nvars)
     cls.col_result = pandas.Series(cls.col_input, index=exog_flattened_idx)
     cls.row_input = np.random.random(nrows)
     cls.row_result = pandas.Series(cls.row_input, index=exog.index)
     cls.cov_input = np.random.random((nvars, nvars))
     cls.cov_result = pandas.DataFrame(cls.cov_input,
                                       index=exog_flattened_idx,
                                       columns=exog_flattened_idx)
     cls.xnames = ['const', 'x_1', 'x_2']
     cls.ynames = 'y_1'
     cls.row_labels = cls.exog.index

    def setup_class(cls):
        cls.endog = np.random.random(10).tolist()

        exog =  pandas.DataFrame(np.random.random((10,2)),
                                 columns=['x_1','x_2'])
        exog.insert(0, 'const', 1)
        cls.exog = exog
        cls.data = sm_data.handle_data(cls.endog, cls.exog)
        nrows = 10
        nvars = 3
        cls.col_input = np.random.random(nvars)
        cls.col_result = pandas.Series(cls.col_input,
                                          index=exog.columns)
        cls.row_input = np.random.random(nrows)
        cls.row_result = pandas.Series(cls.row_input,
                                          index=exog.index)
        cls.cov_input = np.random.random((nvars, nvars))
        cls.cov_result = pandas.DataFrame(cls.cov_input,
                                           index = exog.columns,
                                           columns = exog.columns)
        cls.xnames = ['const', 'x_1', 'x_2']
        cls.ynames = 'y'
        cls.row_labels = cls.exog.index

 def test_labels(self):
     2, 10, 14
     labels = pandas.Index([0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 15,
                            16, 17, 18, 19, 20, 21, 22, 23, 24])
     data = sm_data.handle_data(self.y, self.X, 'drop')
     np.testing.assert_(data.row_labels.equals(labels))

 def test_mv_endog(self):
     y = self.X
     y = y.ix[~np.isnan(y.values).any(axis=1)]
     data = sm_data.handle_data(self.X, None, 'drop')
     np.testing.assert_array_equal(data.endog, y.values)

 def test_endog_only_drop(self):
     y = self.y
     y = y.dropna()
     data = sm_data.handle_data(self.y, None, 'drop')
     np.testing.assert_array_equal(data.endog, y.values)

 def test_none(self):
     data = sm_data.handle_data(self.y, self.X, 'none')
     np.testing.assert_array_equal(data.endog, self.y.values)
     np.testing.assert_array_equal(data.exog, self.X.values)

 def test_raise_no_missing(self):
     # smoke test for #1700
     sm_data.handle_data(pandas.Series(np.random.random(20)),
                         pandas.DataFrame(np.random.random((20, 2))),
                         'raise')

 def test_extra_kwargs_1d(self):
     weights = np.random.random(25)
     data = sm_data.handle_data(self.y, self.X, 'drop', weights=weights)
     idx = ~np.isnan(np.c_[self.y, self.X]).any(axis=1)
     weights = weights[idx]
     np.testing.assert_array_equal(data.weights, weights)

 def test_none(self):
     data = sm_data.handle_data(self.y, self.X, 'none', hasconst=False)
     np.testing.assert_array_equal(data.endog, self.y)
     np.testing.assert_array_equal(data.exog, self.X)

 def test_pandas_noconstant(self):
     exog = self.data.exog.copy()
     data = sm_data.handle_data(self.data.endog, exog)
     np.testing.assert_equal(data.k_constant, 0)
     np.testing.assert_equal(data.const_idx, None)

 def test_array_noconstant(self):
     exog = self.data.exog.copy()
     data = sm_data.handle_data(self.data.endog.values, exog.values)
     np.testing.assert_equal(data.k_constant, 0)
     np.testing.assert_equal(data.const_idx, None)

 def test_pandas_constant(self):
     exog = self.data.exog.copy()
     exog['const'] = 1
     data = sm_data.handle_data(self.data.endog, exog)
     np.testing.assert_equal(data.k_constant, 1)
     np.testing.assert_equal(data.const_idx, 6)

def test_formula_missing_extra_arrays():
    np.random.seed(1)
    # because patsy can't turn off missing data-handling as of 0.3.0, we need
    # separate tests to make sure that missing values are handled correctly
    # when going through formulas

    # there is a handle_formula_data step
    # then there is the regular handle_data step
    # see 2083

    # the untested cases are endog/exog have missing. extra has missing.
    # endog/exog are fine. extra has missing.
    # endog/exog do or do not have missing and extra has wrong dimension
    y = np.random.randn(10)
    y_missing = y.copy()
    y_missing[[2, 5]] = np.nan
    X = np.random.randn(10)
    X_missing = X.copy()
    X_missing[[1, 3]] = np.nan

    weights = np.random.uniform(size=10)
    weights_missing = weights.copy()
    weights_missing[[6]] = np.nan

    weights_wrong_size = np.random.randn(12)

    data = {'y': y,
            'X': X,
            'y_missing': y_missing,
            'X_missing': X_missing,
            'weights': weights,
            'weights_missing': weights_missing}
    data = pandas.DataFrame.from_dict(data)
    data['constant'] = 1

    formula = 'y_missing ~ X_missing'

    ((endog, exog),
     missing_idx, design_info) = handle_formula_data(data, None, formula,
                                                     depth=2,
                                                     missing='drop')

    kwargs = {'missing_idx': missing_idx, 'missing': 'drop',
              'weights': data['weights_missing']}

    model_data = sm_data.handle_data(endog, exog, **kwargs)
    data_nona = data.dropna()
    assert_equal(data_nona['y'].values, model_data.endog)
    assert_equal(data_nona[['constant', 'X']].values, model_data.exog)
    assert_equal(data_nona['weights'].values, model_data.weights)

    tmp = handle_formula_data(data, None, formula, depth=2, missing='drop')
    (endog, exog), missing_idx, design_info = tmp
    weights_2d = np.random.randn(10, 10)
    weights_2d[[8, 7], [7, 8]] = np.nan  #symmetric missing values
    kwargs.update({'weights': weights_2d,
                   'missing_idx': missing_idx})

    model_data2 = sm_data.handle_data(endog, exog, **kwargs)

    good_idx = [0, 4, 6, 9]
    assert_equal(data.loc[good_idx, 'y'], model_data2.endog)
    assert_equal(data.loc[good_idx, ['constant', 'X']], model_data2.exog)
    assert_equal(weights_2d[good_idx][:, good_idx], model_data2.weights)

    tmp = handle_formula_data(data, None, formula, depth=2, missing='drop')
    (endog, exog), missing_idx, design_info = tmp

    kwargs.update({'weights': weights_wrong_size,
                   'missing_idx': missing_idx})
    assert_raises(ValueError, sm_data.handle_data, endog, exog, **kwargs)