def sparse_stable_svd(R, nboot=50):
    # generate the boots
    boots = [np.random.random_integers(0,len(R)-1,len(R))
             for i in xrange(nboot)]

    # calc the original SVD
    U, s, Vh = np.linalg.svd(np.concatenate(R), full_matrices=False)
    
    # do the boots
    rVs = []
    for i in range(len(boots)):
        Ub, sb, Vhb = np.linalg.svd(np.concatenate(R[boots[i]]), full_matrices=False)

        rmat = procrustes(U,Ub)

        rVs.append(np.dot(rmat,np.dot(diagsvd(sb,len(sb),len(sb)),Vhb)))
        
    # get the bootstrap ratios
    rVs = np.array(rVs)
    Vs = np.dot(diagsvd(s,len(s),len(s)),Vh)
    boot_ratio = Vs/rVs.std(0)
    
    # pass the boot ratios through fdrtool to pick stable features
    fachist = np.histogram(boot_ratio.flatten(),bins=500)
    peak = fachist[1][fachist[0]==np.max(fachist[0])][0]
    results = fdrtool.fdrtool(FloatVector(boot_ratio.flatten()-peak), statistic='normal', 
                              plot=False, verbose=False)
    qv = np.array(results.rx('qval')).reshape(boot_ratio.shape)
    #qv = None
    # apply the thresh
    return U,s,Vh,qv,boot_ratio

    def lsaTransform(self,dimensions=1):
        """ Calculate SVD of objects matrix: U . SIGMA . VT = MATRIX 
            Reduce the dimension of sigma by specified factor producing sigma'. 
            Then dot product the matrices:  U . SIGMA' . VT = MATRIX'
        """
        rows,cols= self.matrix.shape

        if dimensions <= rows: #Its a valid reduction

            #Sigma comes out as a list rather than a matrix
            u,sigma,vt = linalg.svd(self.matrix)

            #Dimension reduction, build SIGMA'
            for index in xrange(rows-dimensions, rows):
                sigma[index]=0

            print linalg.diagsvd(sigma,len(self.matrix), len(vt))        

            #Reconstruct MATRIX'
            reconstructedMatrix= dot(dot(u,linalg.diagsvd(sigma,len(self.matrix),len(vt))),vt)

            #Save transform
            self.matrix=reconstructedMatrix

        else:
            print "dimension reduction cannot be greater than %s" % rows

    def __init__(self, data=None, sym=None):
        super(SvdArray, self).__init__(data=data, sym=sym)

        u, s, v = np.linalg.svd(self.x, full_matrices=1)
        self.u, self.s, self.v = u, s, v
        self.sdiag = linalg.diagsvd(s, *x.shape)
        self.sinvdiag = linalg.diagsvd(1./s, *x.shape)

 def train(self):
     # make word-doc vector
     for index, passage in enumerate(self.passages):
         self.__parse(passage, index)
     self.__build(len(self.passages))
     
     print self.matrix.shape
     
     print self
     self.tfidfTransform()
     #print self
     
     # SVD
     self.u, self.sigma, self.vt = linalg.svd(self.matrix)
     print self.u.shape
     print len(self.sigma)
     print self.vt.shape
     
     self.sigma_1 = linalg.diagsvd(self.sigma,len(self.sigma), len(self.sigma)) ** -1
     
     print self.sigma_1
     
     print self.sigma_1 * self.sigma
     
     print linalg.diagsvd(self.sigma,len(self.sigma), len(self.sigma))
     
     # calculate doc concpets
     pass

def image_svd(n):
    img=mpimg.imread('image.jpg')
    [r,g,b] = [img[:,:,i] for i in range(3)]
    r_1,r_2,r_3 = sp.svd(r)
    g_1,g_2,g_3 = sp.svd(g)
    b_1,b_2,b_3 = sp.svd(b)
    r2_nonzero=(r_2!=0).sum()
    g2_nonzero=(g_2!=0).sum()
    b2_nonzero=(b_2!=0).sum()
    print("The number of non zero elements in decompose sigma of red, green, blue matrices are", r2_nonzero,"," ,g2_nonzero,"and" ,b2_nonzero, "respectively.")
    
    r_2[n:800]=np.zeros_like(r_2[n:800])
    g_2[n:800]=np.zeros_like(g_2[n:800])
    b_2[n:800]=np.zeros_like(b_2[n:800])
    
    # change the dimension to (800,1000) 
    r_2=sp.diagsvd(r_2,800,1000)
    g_2=sp.diagsvd(g_2,800,1000)
    b_2=sp.diagsvd(b_2,800,1000)
    
    #dot multiplication
    r_new=np.dot(r_1, np.dot(r_2,r_3))
    g_new=np.dot(g_1, np.dot(g_2,g_3))
    b_new=np.dot(b_1, np.dot(b_2,b_3))

    img[:,:,0]=r_new
    img[:,:,1]=g_new
    img[:,:,2]=b_new
    
    #plot the images
    fig = plt.figure(2)
    ax1 = fig.add_subplot(2,2,1)
    ax2 = fig.add_subplot(2,2,2)
    ax3 = fig.add_subplot(2,2,3)
    ax4 = fig.add_subplot(2,2,4)
    
    ax1.imshow(img)
    ax2.imshow(r, cmap = 'Reds')
    ax3.imshow(g, cmap = 'Greens')
    ax4.imshow(b, cmap = 'Blues')
    plt.show()
    
    #original image
    img=mpimg.imread('image.jpg')
    [r,g,b]=[img[:,:,i] for i in range(3)]
    fig=plt.figure(1)    
    ax1 =  fig.add_subplot(2,2,1)
    ax2 = fig.add_subplot(2,2,2)
    ax3 = fig.add_subplot(2,2,3)
    ax4 = fig.add_subplot(2,2,4)
    ax1.imshow(img)
    ax2.imshow(r, cmap = 'Reds')
    ax3.imshow(g, cmap = 'Greens')
    ax4.imshow(b, cmap = 'Blues')
    plt.show()

    def test_less_accurate_than_full_svd(self):
        A = lowrank(100, 100)

        U, s, Vh = randomized_svd.randomized_svd(A, 10)
        S = la.diagsvd(s, U.shape[1], U.shape[1])
        randomized_err = la.norm(U.dot(S).dot(Vh) - A, 2)

        U, s, Vh = self.full_svd(A)
        S = la.diagsvd(s, U.shape[1], U.shape[1])
        full_err = la.norm(U.dot(S).dot(Vh) - A, 2)

        self.assertGreater(1e-2 * randomized_err, full_err)

def image_svd(n):
    # read image
    img=mpimg.imread('SnakeDance.jpg')

    # generate rgb array
    [r,g,b] = [img[:,:,i] for i in range(3)]
        
    # generate U, sigma,and V for red, green and blue matrix
    #noted that r1=U, r2=sigma, r3=V, same goes to green and blue matrix
    r1, r2, r3 = linalg.svd(r)
    g1, g2, g3 = linalg.svd(g)
    b1, b2, b3 = linalg.svd(b)
    
    #check the number of non zero elements in each color of decompose sigma
    r2_nonzero=(r2!=0).sum()
    g2_nonzero=(g2!=0).sum()
    b2_nonzero=(b2!=0).sum()
    print("The number of non zero elements in decompose sigma of red, green, blue matrices are", r2_nonzero,"," ,g2_nonzero,"and" ,b2_nonzero, "respectively.")
    
    
    # keeping first n none zero elements
    r2[n:800] = np.zeros_like(r2[n:800])
    g2[n:800] = np.zeros_like(g2[n:800])
    b2[n:800] = np.zeros_like(b2[n:800])
    
    # creating diagonal matrix to perform dot multiplication
    #change the dimension of r2 to (800,1000), since original r2 from linalg.svd is (800,1)
    #can check dimension with r2.shape
    r2 = linalg.diagsvd(r2,800,1000)
    g2 = linalg.diagsvd(g2,800,1000)
    b2 = linalg.diagsvd(b2,800,1000)
    
    # perform dot multiplication to create lower resolutuion mariric 
    r_new = np.dot(r1, np.dot(r2, r3))
    g_new = np.dot(g1, np.dot(g2, g3))
    b_new = np.dot(b1, np.dot(b2, b3))
      
    img[:,:,0]=r_new
    img[:,:,1]=g_new
    img[:,:,2]=b_new
    
    fig2 = plt.figure(2)
    ax1 = fig2.add_subplot(2,2,1)
    ax2 = fig2.add_subplot(2,2,2)
    ax3 = fig2.add_subplot(2,2,3)
    ax4 = fig2.add_subplot(2,2,4)
    ax1.imshow(img)
    ax2.imshow(r_new, cmap = 'Reds')
    ax3.imshow(g_new, cmap = 'Greens')
    ax4.imshow(b_new, cmap = 'Blues')
    plt.show() 

def low_rank_approx(X,r):
    U, s, Vh = linalg.svd(X)
    s [r:] = 0
    sk = linalg.diagsvd(s, U.shape[1], Vh.shape[0])
    X_app = np.dot(U, np.dot(sk, Vh))
    X_app = X_app[:,:r]
    return X_app

def pca(X):
    #PCA Run principal component analysis on the dataset X
    #   [U, S, X] = pca(X) computes eigenvectors of the covariance matrix of X
    #   Returns the eigenvectors U, the eigenvalues (on diagonal) in S
    #

    # Useful values
    m, n = X.shape

    # You need to return the following variables correctly.
    U = np.zeros(n)
    S = np.zeros(n)

    # ====================== YOUR CODE HERE ======================
    # Instructions: You should first compute the covariance matrix. Then, you
    #               should use the "svd" function to compute the eigenvectors
    #               and eigenvalues of the covariance matrix. 
    #
    # Note: When computing the covariance matrix, remember to divide by m (the
    #       number of examples).
    #

    # compute the covariance matrix
    sigma = (1.0/m) * (X.T).dot(X)

    # compute the eigenvectors (U) and S
    # from: 
    # http://docs.scipy.org/doc/scipy/reference/generated/scipy.linalg.svd.html#scipy.linalg.svd
    # http://docs.scipy.org/doc/scipy/reference/generated/scipy.linalg.diagsvd.html#scipy.linalg.diagsvd
    U, S, Vh = linalg.svd(sigma)
    S = linalg.diagsvd(S, len(S), len(S))

    # =========================================================================

    return U, S

def matrix_reduce_sigma(matrix, dimensions=1):
    """This calculates the SVD of the matrix, reduces it and 
        creates a reduced matrix.

        @params matrix the matrix to reduce
        @params dimensions dimensions to reduce. 

        @return matrix The reduced matrix
    """
    uu, sigma, vt = linalg.svd(matrix)
    rows = sigma.shape[0]
    cols = sigma.shape[1]

    #delete n-k smallest singular values 
    #delete ie settings to zero
    smallerBound = min(rows, cols)
    for index in xrange(smallerBound - dimensions, rows):
        sigma[index] = 0 
    
    #since sigma is a unidimensional array
    #convert it to a matrix 
    sigma_matrix = linalg.diagsvd(sigma, len(uu), len(vt))
    uu_sigma = numpy.dot(uu, sigma_matrix)
    uu_sigma_vt = numpy.dot(uu_sigma, vt)

    return uu_sigma_vt

    def fs_c(self, percent=0.9, N=None):
        """Get the column factor scores (dimensionality-reduced representation),
        choosing how many factors to retain, directly or based on the explained
        variance.

        'percent': The minimum variance that the retained factors are required
                                to explain (default: 90% = 0.9)
        'N': The number of factors to retain. Overrides 'percent'.
                If the rank is less than N, N is ignored.
        """
        if not 0 <= percent <= 1:
                raise ValueError("Percent should be a real number between 0 and 1.")
        if N:
                if not isinstance(N, (int, np.int64)) or N <= 0:
                        raise ValueError("N should be a positive integer.")
                N = min(N, self.rank)  # maybe we should notify the user?
                # S = np.zeros((self._numitems, N))
        # else:
        self.k = 1 + np.flatnonzero(np.cumsum(self.L) >= sum(self.L)*percent)[0]
        #  S = np.zeros((self._numitems, self.k))
        # the sign of the square root can be either way; singular value vs. eigenvalue
        # np.fill_diagonal(S, -np.sqrt(self.E) if self.cor else self.s)
        num2ret = N if N else self.k
        s = -np.sqrt(self.L) if self.cor else self.s
        S = diagsvd(s[:num2ret], len(self.Q), num2ret)
        self.G = _mul(self.D_c, self.Q.T, S)  # important! note the transpose on Q
        return self.G

    def fs_r(self, percent=0.9, N=None):
        """Get the row factor scores (dimensionality-reduced representation),
        choosing how many factors to retain, directly or based on the explained
        variance.

        'percent': The minimum variance that the retained factors are required
                                to explain (default: 90% = 0.9)
        'N': The number of factors to retain. Overrides 'percent'.
                If the rank is less than N, N is ignored.
        """
        if not 0 <= percent <= 1:
                raise ValueError("Percent should be a real number between 0 and 1.")
        if N:
                if not isinstance(N, (int, np.int64)) or N <= 0:
                        raise ValueError("N should be a positive integer.")
                N = min(N, self.rank)
                # S = np.zeros((self._numitems, N))
        # else:
        self.k = 1 + np.flatnonzero(np.cumsum(self.L) >= sum(self.L)*percent)[0]
        #  S = np.zeros((self._numitems, self.k))
        # the sign of the square root can be either way; singular value vs. eigenvalue
        # np.fill_diagonal(S, -np.sqrt(self.E) if self.cor else self.s)
        num2ret = N if N else self.k
        s = -np.sqrt(self.L) if self.cor else self.s
        S = diagsvd(s[:num2ret], self._numitems, num2ret)

        from numpy import ndarray
        if not isinstance(self.D_r, ndarray):
            self.F = self.D_r.dot(self.P).dot(S[:self.P.shape[1]])
        else:
            self.F = _mul(self.D_r, self.P, S)
        return self.F

 def svd(self, matrix):
   matrix = numpy.mat(matrix);
   self._U_, self._SIGMA_, self._Vh_ = linalg.svd(matrix);
   #perform the SVD 
   self.M, self.N = matrix.shape;
   Sig = numpy.mat(linalg.diagsvd(self._SIGMA_, self.M, self.N)) 
   print Sig

def special_svd(M, K=9):
    useravg, itemavg = find_user_and_item_avg(M)
    R_norm = norm_matrix(M, useravg, itemavg)
    U, s, V = linalg.svd( R_norm, full_matrices = False)
    new_s = s[:K]
    sigma = linalg.diagsvd(new_s, K, K)
    return U[:,:K], V[:K,:], sigma

def multivariateGaussian(X, mu, sigma2):
    #MULTIVARIATEGAUSSIAN Computes the probability density function of the
    #multivariate gaussian distribution.
    #    p = MULTIVARIATEGAUSSIAN(X, mu, sigma2) Computes the probability 
    #    density function of the examples X under the multivariate gaussian 
    #    distribution with parameters mu and sigma2. If sigma2 is a matrix, it is
    #    treated as the covariance matrix. If sigma2 is a vector, it is treated
    #    as the \sigma^2 values of the variances in each dimension (a diagonal
    #    covariance matrix)
    #

    k = len(mu)

    # turns 1D array into 2D array
    if sigma2.ndim == 1:
        sigma2 = np.reshape(sigma2, (-1,sigma2.shape[0]))

    if sigma2.shape[1] == 1 or sigma2.shape[0] == 1:
        sigma2 = linalg.diagsvd(sigma2.flatten(), len(sigma2.flatten()), len(sigma2.flatten()))

    # mu is unrolled (and transposed) here
    X = X - mu.reshape(mu.size, order='F').T

    p = np.dot(np.power(2 * np.pi, - k / 2.0), np.power(np.linalg.det(sigma2), -0.5) ) * \
        np.exp(-0.5 * np.sum(np.dot(X, np.linalg.pinv(sigma2)) * X, axis=1))

    return p

def fillmat(M):
    m, n = M.shape
    X = np.zeros(shape=(m, n))    
    tau = 1.0    
    mu_min = 1.0e-8
    eta_mu = 0.25
    mu = eta_mu * norm(np.nan_to_num(M)) 

    niter = 0
    max_iter = 10000
    xtol = 1.0e-3

    while (mu > mu_min) and (niter < max_iter):
        delta = 1.0
        while delta > xtol:         
            X_prev = X
            Y = X - tau * np.nan_to_num(X - M)
            U, S, V = svd(Y, full_matrices=False)
            S1 = np.maximum(S - tau * mu, 0)
            S1 = diagsvd(S1, n, n)
            X = np.dot(U, np.dot(S1, V))
            delta = get_error(X, X_prev)            

        mu = max(mu * eta_mu, mu_min)
        niter += 1       
        print 'mu = {:0.4e}'.format(mu)

    return X

    def __init__(self, corpus, vocab):
        """
      Create CountVectorizer object,
      Create a tfidf array
      Use SVD (Singular Value Decomposition) to approximate tfidf array
      Pickle-able
    """

        self.v = CountVectorizer(vocabulary=vocab)

        X = self.v.fit_transform(corpus).toarray()

        transformer = TfidfTransformer()

        tfidf = transformer.fit_transform(X)

        # SVD
        M, N = X.shape

        U, s, Vt = linalg.svd(X)

        # Reduce Matrix to only 300 dimensions
        for i in range(len(s)):
            if i < 300:
                continue
            s[i] = 0

        Sig = linalg.diagsvd(s, M, N)

        print U.shape
        print Sig.shape
        print Vt.shape

        # Store approximated document-term Matrix
        self.dt = (U.dot(Sig.dot(Vt))).transpose()

def plotFirst3PCA(X, labels=None, colors=None):
    '''
    Computes the first 3 principal components of the data
    matrix X, and shows the samples projected onto the 3 largest
    components using scatter3d()
    @param X: Input data, samples are in rows. It is advised to
    at least mean-center the data, but also to scale each input feature
    by dividing by standard deviation. Use svo_util.normalize() to
    do this.
    @param labels: A vector with length = rows(X), which has an integer
    label that indicates which class each sample belongs to. None means
    that the data is not classified, so all points will have the same
    color.
    @param colors: A list of color strings or numbers,
    one per label so that all points with the same label
    are colored the same. len(colors) == len( unique(labels) )
    @return: (T, W) where T is the data in pca-space and W are the
    loading weights. T and W can be used to reconstruct points from
    PCA space back to the 'normal' space, as with the function
    reconstructPCA().
    '''
    U,s,Vt = LA.svd(X, full_matrices=True)
    N,p = X.shape
    S = LA.diagsvd(s,N,p)
    T = U.dot(S)  #samples in PCA space (also, T = X.dot(V) where V=Vt.T)
    
    XYZ = T[:,0:3]  #first 3 columns are for the 3 largest components
    scatter3d(XYZ, labels=labels, colors=colors)
    
    return T, Vt.T  #return the transformed data, and the loading weights

def image_svd(n):
    img=mpimg.imread('mypicture.jpg')
    [r,g,b] = [img[:,:,i] for i in range(3)]
    r1,r2,r3 = sp.svd(r)
    g1,g2,g3 = sp.svd(g)
    b1,b2,b3 = sp.svd(b)
    r2_nonzero=(r2!=0).sum()
    g2_nonzero=(g2!=0).sum()
    b2_nonzero=(b2!=0).sum()
    print("The number of non zero elements in decompose sigma of red, green, blue matrices are", r2_nonzero,"," ,g2_nonzero,"and" ,b2_nonzero, "respectively.")
    
    r2[n:800]=np.zeros_like(r2[n:800])
    g2[n:800]=np.zeros_like(g2[n:800])
    b2[n:800]=np.zeros_like(b2[n:800])
    r2=sp.diagsvd(r2,800,1000)
    g2=sp.diagsvd(g2,800,1000)
    b2=sp.diagsvd(b2,800,1000)
    r_new=np.dot(r1, np.dot(r2,r3))
    g_new=np.dot(g1, np.dot(g2,g3))
    b_new=np.dot(b1, np.dot(b2,b3))
    img[:,:,0]=r_new
    img[:,:,1]=g_new
    img[:,:,2]=b_new
    
    fig = plt.figure(2)
    ax1 = fig.add_subplot(2,2,1)
    ax2 = fig.add_subplot(2,2,2)
    ax3 = fig.add_subplot(2,2,3)
    ax4 = fig.add_subplot(2,2,4)
    ax1.imshow(img)
    ax2.imshow(r, cmap = 'Reds')
    ax3.imshow(g, cmap = 'Greens')
    ax4.imshow(b, cmap = 'Blues')
    plt.show()
    
    img=mpimg.imread('mypicture.jpg')
    [r,g,b]=[img[:,:,i] for i in range(3)]
    fig=plt.figure(1)
    ax1=fig.add_subplot(2,2,1)
    ax2 = fig.add_subplot(2,2,2)
    ax3 = fig.add_subplot(2,2,3)
    ax4 = fig.add_subplot(2,2,4)
    ax1.imshow(img)
    ax2.imshow(r, cmap = 'Reds')
    ax3.imshow(g, cmap = 'Greens')
    ax4.imshow(b, cmap = 'Blues')
    plt.show()

def svd(R):
	'''
	Returns singular value decomposition of the ratings matrix
	'''
	U, S, Vt = linalg.svd(R, full_matrices=False)
	k = len(S) 
	S = linalg.diagsvd(S, k, k)
	return U, S, Vt