def load(cls, sc, path):
     """Load a IsotonicRegressionModel."""
     java_model = sc._jvm.org.apache.spark.mllib.regression.IsotonicRegressionModel.load(
         sc._jsc.sc(), path)
     py_boundaries = _java2py(sc, java_model.boundaryVector()).toArray()
     py_predictions = _java2py(sc, java_model.predictionVector()).toArray()
     return IsotonicRegressionModel(py_boundaries, py_predictions, java_model.isotonic)

 def load(cls, sc, path):
     java_model = sc._jvm.org.apache.spark.mllib.classification.NaiveBayesModel.load(
         sc._jsc.sc(), path)
     py_labels = _java2py(sc, java_model.labels())
     py_pi = _java2py(sc, java_model.pi())
     py_theta = _java2py(sc, java_model.theta())
     return NaiveBayesModel(py_labels, py_pi, numpy.array(py_theta))

    def __init__(self, regressionCoeff=None, d=0, q=0, coefficients=None, hasIntercept=False, jmodel=None, sc=None):
        """
        Parameters
        ----------
        regressionCoeff:
            coefficients for regression , including intercept , for example. if model has 3 regressors
            then length of [[regressionCoeff]] is 4
        arimaOrders:
            p,d,q for the arima error structure, length of [[arimaOrders]] must be 3
        arimaCoeff:
            AR, d, and MA terms, length of arimaCoeff = p+d+q
        """
        assert sc != None, "Missing SparkContext"

        self._ctx = sc
        if jmodel == None:
            self._jmodel = self._ctx._jvm.com.cloudera.sparkts.models.RegressionARIMAModel(
                _py2java_double_array(self._ctx, regressionCoeff),
                _py2java_int_array(self._ctx, arimaOrders),
                _py2scala_arraybuffer(self._ctx, arimaCoeff),
            )
        else:
            self._jmodel = jmodel

        self.regressionCoeff = _java2py(sc, self._jmodel.regressionCoeff())
        self.arimaOrders = _java2py(sc, self._jmodel.arimaOrders())
        self.arimaCoeff = _java2py(sc, self._jmodel.arimaCoeff())

 def load(cls, sc, path):
     java_model = sc._jvm.org.apache.spark.mllib.classification.NaiveBayesModel.load(
         sc._jsc.sc(), path)
     # Can not unpickle array.array from Pyrolite in Python3 with "bytes"
     py_labels = _java2py(sc, java_model.labels(), "latin1")
     py_pi = _java2py(sc, java_model.pi(), "latin1")
     py_theta = _java2py(sc, java_model.theta(), "latin1")
     return NaiveBayesModel(py_labels, py_pi, numpy.array(py_theta))

 def __init__(self, p=0, d=0, q=0, coefficients=None, hasIntercept=False, jmodel=None, sc=None):
     self._ctx = sc
     if jmodel == None:
         self._jmodel = self._ctx._jvm.com.cloudera.sparkts.models.ARIMAModel(p, d, q, _py2java(self._ctx, coefficients), hasIntercept)
     else:
         self._jmodel = jmodel
         
     self.p = _java2py(sc, self._jmodel.p())
     self.d = _java2py(sc, self._jmodel.d())
     self.q = _java2py(sc, self._jmodel.q())
     self.coefficients = _java2py(sc, self._jmodel.coefficients())
     self.has_intercept = _java2py(sc, self._jmodel.hasIntercept())

    def __init__(self, p=0, d=0, q=0, coefficients=None, hasIntercept=True, jmodel=None, sc=None):
        assert sc != None, "Missing SparkContext"

        self._ctx = sc
        if jmodel == None:
            self._jmodel = self._ctx._jvm.com.cloudera.sparkts.models.ARIMAModel(p, d, q, _py2java_double_array(self._ctx, coefficients), hasIntercept)
        else:
            self._jmodel = jmodel
            
        self.p = _java2py(sc, self._jmodel.p())
        self.d = _java2py(sc, self._jmodel.d())
        self.q = _java2py(sc, self._jmodel.q())
        self.coefficients = _java2py(sc, self._jmodel.coefficients())
        self.has_intercept = _java2py(sc, self._jmodel.hasIntercept())

 def gradient_log_likelihood_css_arma(self, diffedy):
     """
     Calculates the gradient for the log likelihood function using CSS
     Derivation:
         L(y | \theta) = -\frac{n}{2}log(2\pi\sigma^2) - \frac{1}{2\pi}\sum_{i=1}^n \epsilon_t^2 \\
         \sigma^2 = \frac{\sum_{i = 1}^n \epsilon_t^2}{n} \\
         \frac{\partial L}{\partial \theta} = -\frac{1}{\sigma^2}
         \sum_{i = 1}^n \epsilon_t \frac{\partial \epsilon_t}{\partial \theta} \\
         \frac{\partial \epsilon_t}{\partial \theta} = -\frac{\partial \hat{y}}{\partial \theta} \\
         \frac{\partial\hat{y}}{\partial c} = 1 +
         \phi_{t-q}^{t-1}*\frac{\partial \epsilon_{t-q}^{t-1}}{\partial c} \\
         \frac{\partial\hat{y}}{\partial \theta_{ar_i}} =  y_{t - i} +
         \phi_{t-q}^{t-1}*\frac{\partial \epsilon_{t-q}^{t-1}}{\partial \theta_{ar_i}} \\
         \frac{\partial\hat{y}}{\partial \theta_{ma_i}} =  \epsilon_{t - i} +
         \phi_{t-q}^{t-1}*\frac{\partial \epsilon_{t-q}^{t-1}}{\partial \theta_{ma_i}} \\
     
     Parameters
     ----------
     diffedY:
         array of differenced values
     
     returns the gradient log likelihood as an array of double
     """
     # need to copy diffedy to a double[] for Java
     result =  self._jmodel.gradientlogLikelihoodCSSARMA(_py2java_double_array(self._ctx, diffedy))
     return _java2py(self._ctx, result)

 def load(cls, sc, path):
     java_model = sc._jvm.org.apache.spark.mllib.regression.RidgeRegressionModel.load(
         sc._jsc.sc(), path)
     weights = _java2py(sc, java_model.weights())
     intercept = java_model.intercept()
     model = RidgeRegressionModel(weights, intercept)
     return model

def _call_java(sc, java_obj, name, *args):
    """
    Method copied from pyspark.ml.wrapper.  Uses private Spark APIs.
    """
    m = getattr(java_obj, name)
    java_args = [_py2java(sc, arg) for arg in args]
    return _java2py(sc, m(*java_args))

 def forecast(self, ts, nfuture):
     """
     Provided fitted values for timeseries ts as 1-step ahead forecasts, based on current
     model parameters, and then provide `nFuture` periods of forecast. We assume AR terms
     prior to the start of the series are equal to the model's intercept term (or 0.0, if fit
     without and intercept term).Meanwhile, MA terms prior to the start are assumed to be 0.0. If
     there is differencing, the first d terms come from the original series.
    
     Parameters
     ----------
     ts:
         Timeseries to use as gold-standard. Each value (i) in the returning series
         is a 1-step ahead forecast of ts(i). We use the difference between ts(i) -
         estimate(i) to calculate the error at time i, which is used for the moving
         average terms. Numpy array.
     nFuture:
         Periods in the future to forecast (beyond length of ts)
         
     Returns a series consisting of fitted 1-step ahead forecasts for historicals and then
     `nFuture` periods of forecasts. Note that in the future values error terms become
     zero and prior predictions are used for any AR terms.
     
     """
     jts = _py2java(self._ctx, Vectors.dense(ts))
     jfore = self._jmodel.forecast(jts, nfuture)
     return _java2py(self._ctx, jfore)

def perform_pca(matrix, row_count, nr_principal_components=2):
    """Return principal components of the input matrix.

    This function uses MLlib's ``RowMatrix`` to compute principal components.

    Args:
        matrix: An RDD[int, (int, float)] representing a sparse matrix. This
            is returned by ``center_matrix`` but it is not required to center
            the matrix first.
        row_count: The size (N) of the N x N ``matrix``.
        nr_principal_components: Number of components we want to obtain. This
            value must be less than or equal to the number of rows in the input
            square matrix.

    Returns:
        An array of ``nr_principal_components`` columns, and same number of rows
        as the input ``matrix``. This array is a ``numpy`` array.
    """

    py_rdd = matrix.map(lambda row: linalg.Vectors.sparse(row_count, row))
    sc = pyspark.SparkContext._active_spark_context
    java_rdd = mllib_common._py2java(sc, py_rdd)
    scala_rdd = java_rdd.rdd()
    sc = pyspark.SparkContext._active_spark_context
    row_matrix = (sc._jvm.org.apache.spark.mllib.linalg.distributed.
        RowMatrix(scala_rdd)
    )
    pca = row_matrix.computePrincipalComponents(nr_principal_components)
    pca = mllib_common._java2py(sc, pca)
    return pca.toArray()

 def log_likelihood(self, ts):
     """
     Returns the log likelihood of the parameters on the given time series.
     
     Based on http://www.unc.edu/~jbhill/Bollerslev_GARCH_1986.pdf
     """
     likelihood = self._jmodel.logLikelihood(_py2java(self._ctx, Vectors.dense(ts)))
     return _java2py(self._ctx, likelihood)

 def to_double_rdd(self, column_index):
     """
     Returns a RDD by converting values to double of given column index
     :param column_index:One column index in TransformableRDD
     :return:RDD
     """
     rdd = self._transformable_rdd.toDoubleRDD(column_index).rdd()
     return _java2py(self.spark_context, rdd)

 def multiply_columns(self, first_column, second_column):
     """
     Returns a RDD which is a product of the values in @first_column and @second_column
     :param first_column: One column index
     :param second_column: Another column index
     :return: RDD
     """
     _rdd = self._transformable_rdd.multiplyColumns(first_column, second_column).rdd()
     return _java2py(self.spark_context, _rdd)

 def smooth(self, column_index, smoothing_method):
     """
     Returns a new RDD containing smoothed values of @column_index using @smoothing_method
     :param column_index: Index of the column
     :param smoothing_method: smoothing method by which you want to smooth data
     :return: RDD
     """
     method = smoothing_method._get_smoothing_method(self.spark_context)
     rdd = self._transformable_rdd.smooth(column_index, method)
     return _java2py(self.spark_context, rdd.rdd())

 def gradient(self, ts):
     """
     Find the gradient of the log likelihood with respect to the given time series.
     
     Based on http://www.unc.edu/~jbhill/Bollerslev_GARCH_1986.pdf
     
     Returns an 3-element array containing the gradient for the alpha, beta, and omega parameters.
     """
     gradient = self._jmodel.gradient(_py2java(self._ctx, Vectors.dense(ts)))
     return _java2py(self._ctx, gradient)

 def _transfer_params_from_java(self):
     """
     Transforms the embedded params from the companion Java object.
     """
     sc = SparkContext._active_spark_context
     for param in self.params:
         if self._java_obj.hasParam(param.name):
             java_param = self._java_obj.getParam(param.name)
             value = _java2py(sc, self._java_obj.getOrDefault(java_param))
             self._paramMap[param] = value

 def _transfer_param_map_from_java(self, javaParamMap):
     """
     Transforms a Java ParamMap into a Python ParamMap.
     """
     sc = SparkContext._active_spark_context
     paramMap = dict()
     for pair in javaParamMap.toList():
         param = pair.param()
         if self.hasParam(str(param.name())):
             paramMap[self.getParam(param.name())] = _java2py(sc, pair.value())
     return paramMap

 def get(self, *outputs):
     """
     Parameters
     ----------
     outputs: string, list of strings
         Output variables as defined inside the DML script.
     """
     outs = [_java2py(self.sc, self._java_results.get(out)) for out in outputs]
     if len(outs) == 1:
         return outs[0]
     return outs

 def __init__(self, c=0, coefficients=None, jmodel=None, sc=None):
     assert sc != None, "Missing SparkContext"
     
     self._ctx = sc
     if jmodel == None:
         self._jmodel = self._ctx._jvm.com.cloudera.sparkts.models.ARModel(c, _py2java_double_array(self._ctx, coefficients))
     else:
         self._jmodel = jmodel
     
     self.c = self._jmodel.c()
     self.coefficients = _java2py(self._ctx, self._jmodel.coefficients())