def _cells_to_rects(self, cells):
        """
        Converts the extents of a list of cell grid coordinates (i,j) into
        a list of rect tuples (x,y,w,h).  The set should be disjoint, but may
        or may not be minimal.
        """
        # Since this function is generally used to generate clipping regions
        # or other screen-related graphics, we should try to return large
        # rectangular blocks if possible.
        # For now, we just look for horizontal runs and return those.
        cells = array(cells)
        y_sorted = sort_points(cells, index=1)  # sort acoording to row
        rownums = sort(array(tuple(set(cells[:,1]))))

        row_start_indices = searchsorted(y_sorted[:,1], rownums)
        row_end_indices = left_shift(row_start_indices, len(cells))

        rects = []
        for rownum, start, end in zip(rownums, row_start_indices, row_end_indices):
            # y_sorted is sorted by the J (row) coordinate, so after we
            # extract the column indices, we need to sort them before
            # passing them to find_runs().
            grid_column_indices = sort(y_sorted[start:end][:,0])
            #pdb.set_trace()
            #print grid_column_indices.shape
            for span in find_runs(grid_column_indices):
                x = self._cell_lefts[span[0]]
                y = self._cell_bottoms[rownum]
                w = (span[-1] - span[0] + 1) * self._cell_extents[0]
                h = self._cell_extents[1]
                rects.append((x,y,w,h))
        return rects

def plot_raw_data(ratings):
    """plot the statistics result on raw rating data."""
    # do statistics.
    num_items_per_user = np.array((ratings != 0).sum(axis=0)).flatten()
    num_users_per_item = np.array((ratings != 0).sum(axis=1).T).flatten()
    sorted_num_movies_per_user = np.sort(num_items_per_user)[::-1]
    sorted_num_users_per_movie = np.sort(num_users_per_item)[::-1]

    # plot
    fig = plt.figure()
    ax1 = fig.add_subplot(1, 2, 1)
    ax1.plot(sorted_num_movies_per_user, color='blue')
    ax1.set_xlabel("users")
    ax1.set_ylabel("number of ratings (sorted)")
    ax1.grid()

    ax2 = fig.add_subplot(1, 2, 2)
    ax2.plot(sorted_num_users_per_movie)
    ax2.set_xlabel("items")
    ax2.set_ylabel("number of ratings (sorted)")
    ax2.set_xticks(np.arange(0, 2000, 300))
    ax2.grid()

    plt.tight_layout()
    plt.savefig("stat_ratings")
    plt.show()
    # plt.close()
    return num_items_per_user, num_users_per_item

def quantiles(x, qlist=(2.5, 25, 50, 75, 97.5), transform=lambda x: x):
    R"""Returns a dictionary of requested quantiles from array

    Parameters
    ----------
    x : Numpy array
        An array containing MCMC samples
    qlist : tuple or list
        A list of desired quantiles (defaults to (2.5, 25, 50, 75, 97.5))
    transform : callable
        Function to transform data (defaults to identity)

    Returns
    -------
    `dictionary` with the quantiles {quantile: value}
    """
    # Make a copy of trace
    x = transform(x.copy())

    # For multivariate node
    if x.ndim > 1:
        # Transpose first, then sort, then transpose back
        sx = np.sort(x.T).T
    else:
        # Sort univariate node
        sx = np.sort(x)

    try:
        # Generate specified quantiles
        quants = [sx[int(len(sx) * q / 100.0)] for q in qlist]

        return dict(zip(qlist, quants))

    except IndexError:
        pm._log.warning("Too few elements for quantile calculation")

    def test_multiindex_objects(self):
        mi = MultiIndex(levels=[['b', 'd', 'a'], [1, 2, 3]],
                        labels=[[0, 1, 0, 2], [2, 0, 0, 1]],
                        names=['col1', 'col2'])
        recons = mi._sort_levels_monotonic()

        # these are equal
        assert mi.equals(recons)
        assert Index(mi.values).equals(Index(recons.values))

        # _hashed_values and hash_pandas_object(..., index=False)
        # equivalency
        expected = hash_pandas_object(
            mi, index=False).values
        result = mi._hashed_values
        tm.assert_numpy_array_equal(result, expected)

        expected = hash_pandas_object(
            recons, index=False).values
        result = recons._hashed_values
        tm.assert_numpy_array_equal(result, expected)

        expected = mi._hashed_values
        result = recons._hashed_values

        # values should match, but in different order
        tm.assert_numpy_array_equal(np.sort(result),
                                    np.sort(expected))

    def test_weaklimit(self):
        a = distributions.CRP(10,1)
        b = distributions.GammaCompoundDirichlet(1000,10,1)

        a.concentration = b.concentration = 10.

        from matplotlib import pyplot as plt

        plt.figure()
        crp_counts = np.zeros(10)
        gcd_counts = np.zeros(10)
        for itr in range(500):
            crp_rvs = np.sort(a.rvs(25))[::-1][:10]
            crp_counts[:len(crp_rvs)] += crp_rvs
            gcd_counts += np.sort(b.rvs(25))[::-1][:10]

        plt.plot(crp_counts/200,gcd_counts/200,'bx-')
        plt.xlim(0,10)
        plt.ylim(0,10)

        import os
        from mixins import mkdir
        figpath = os.path.join(os.path.dirname(__file__),'figures',
                self.__class__.__name__,'weaklimittest.pdf')
        mkdir(os.path.dirname(figpath))
        plt.savefig(figpath)

    def test_mass_grid(self):
        """
        Check that the mass-based grid is constructed correctly.
        """
        ## Test typical input - should be sorted
        levels = utl.define_density_mass_grid(self.unique_density)
        answer = np.sort(self.unique_density)
        assert_array_equal(answer, levels)

        ## Test more levels than density values (answer is the same as typical
        #  input).
        levels = utl.define_density_mass_grid(self.unique_density,
                                              num_levels=self.n * 2)
        assert_array_equal(answer, levels)

        ## Test fewer levels than density values.
        levels = utl.define_density_mass_grid(self.unique_density,
                                              num_levels=2)
        answer = np.array([1, 10])
        assert_array_equal(answer, levels)

        ## Test negative values.
        levels = utl.define_density_mass_grid(self.generic_array)
        answer = np.sort(self.generic_array)
        assert_array_equal(answer, levels)

        ## Test uniform input.
        levels = utl.define_density_mass_grid(self.uniform_density)
        self.assertItemsEqual(levels, [1.])

def quantiles(x, qlist=(2.5, 25, 50, 75, 97.5)):
    """Returns a dictionary of requested quantiles from array

    :Arguments:
      x : Numpy array
          An array containing MCMC samples
      qlist : tuple or list
          A list of desired quantiles (defaults to (2.5, 25, 50, 75, 97.5))

    """

    # Make a copy of trace
    x = x.copy()

    # For multivariate node
    if x.ndim > 1:
        # Transpose first, then sort, then transpose back
        sx = np.sort(x.T).T
    else:
        # Sort univariate node
        sx = np.sort(x)

    try:
        # Generate specified quantiles
        quants = [sx[int(len(sx)*q/100.0)] for q in qlist]

        return dict(zip(qlist, quants))

    except IndexError:
        print("Too few elements for quantile calculation")

def read_multivector_griddata_ascii(name_or_obj):
    """Read 2-d grid data from a text file.

    Each line has values `x0 x1 y0 y1 ...`. Space separated.
    Assumed to be grid of values.

    Parameters
    ----------
    name_or_obj : str or file-like object
        The name of the file or a file-like object containing the
        data.

    Returns
    -------
    x0 : numpy.ndarray
        1-d array.
    x1 : numpy.ndarray
        1-d array.
    y : numpy.ndarray
        3-d array of shape ``(n, len(x0), len(x1))`` where ``n`` is
        the number of y values on each line.
    """
    data = np.loadtxt(name_or_obj)

    x0 = np.sort(np.unique(data[:, 0]))
    x1 = np.sort(np.unique(data[:, 1]))
    y = np.zeros((len(data[0]) - 2, len(x0), len(x1)))

    for i0, p in enumerate(x0):
        for i1, q in enumerate(x1):
            ind = (data[:, 0] == p) & (data[:, 1] == q)
            y[:, i0, i1] = data[ind, 2:]

    return x0, x1, y

def regenerate_dim(x):
    """ assume x in ns since epoch from the current time """
    msg = None  # msg allows us to see which shot/diag was at fault
    diffs = np.diff(x)
    # bincount needs a positive input and needs an array with N elts where N is the largest number input
    small = (diffs > 0) & (diffs < 1000000)
    sorted_diffs = np.sort(diffs[np.where(small)[0]])
    counts = np.bincount(sorted_diffs)
    bigcounts, bigvals = myhist(diffs[np.where(~small)[0]])

    if pyfusion.VERBOSE>0:
        print('[[diff, count],....]')
        print('small:', [[argc, counts[argc]] for argc in np.argsort(counts)[::-1][0:5]])
        print('big or negative:', [[bigvals[argc], bigcounts[argc]] for argc in np.argsort(bigcounts)[::-1][0:10]])

    dtns = 1 + np.argmax(counts[1:])  # skip the first position - it is 0
    # wgt0 = np.where(sorted_diffs > 0)[0]  # we are in ns, so no worry about rounding
    histo = plt.hist if pyfusion.DBG() > 1 else np.histogram
    cnts, vals = histo(x, bins=200)[0:2]
    # ignore the two end bins - hopefully there will be very few there
    wmin = np.where(cnts[1:-1] < np.max(cnts[1:-1]))[0]
    if len(wmin)>0:
        print('**********\n*********** Gap in data > {p:.2f}%'.format(p=100*len(wmin)/float(len(cnts))))
    x01111 = np.ones(len(x))  # x01111 will be all 1s except for the first elt.
    x01111[0] = 0
    errcnt = np.sum(bigcounts) + np.sum(np.sort(counts)[::-1][1:])
    if errcnt>0 or (pyfusion.VERBOSE > 0): 
        msg = str('** repaired length of {l:,}, dtns={dtns:,}, {e} erroneous utcs'
              .format(l=len(x01111), dtns=dtns, e=errcnt))

    fixedx = np.cumsum(x01111)*dtns
    wbad = np.where((x - fixedx)>1e8)[0]
    fixedx[wbad] = np.nan
    debug_(pyfusion.DEBUG, 3, key="repair", msg="repair of W7-X scrambled Langmuir timebase") 
    return(fixedx, msg)

def test_calculate_landslide_probability_lognormal_method():
    """Testing the main method 'calculate_landslide_probability()' with
    'lognormal' method. 
    """
    grid_2 = RasterModelGrid((5, 4), spacing=(0.2, 0.2))
    gridnum = grid_2.number_of_nodes
    np.random.seed(seed=6)
    grid_2.at_node['topographic__slope'] = np.random.rand(gridnum)
    scatter_dat = np.random.randint(1, 10, gridnum)
    grid_2.at_node['topographic__specific_contributing_area']= (
             np.sort(np.random.randint(30, 900, gridnum)))
    grid_2.at_node['soil__transmissivity']= (
             np.sort(np.random.randint(5, 20, gridnum), -1))
    grid_2.at_node['soil__mode_total_cohesion']= (
             np.sort(np.random.randint(30, 900, gridnum)))
    grid_2.at_node['soil__minimum_total_cohesion']= (
             grid_2.at_node['soil__mode_total_cohesion'] - scatter_dat)
    grid_2.at_node['soil__maximum_total_cohesion']= (
             grid_2.at_node['soil__mode_total_cohesion'] + scatter_dat)
    grid_2.at_node['soil__internal_friction_angle']= (
             np.sort(np.random.randint(26, 37, gridnum)))
    grid_2.at_node['soil__thickness']= (
             np.sort(np.random.randint(1, 10, gridnum)))
    grid_2.at_node['soil__density']= (2000. * np.ones(gridnum))

    ls_prob_lognormal = LandslideProbability(grid_2, number_of_iterations=10,
        groundwater__recharge_distribution='lognormal',
        groundwater__recharge_mean=5.,
        groundwater__recharge_standard_deviation=0.25,
        seed=6)
    ls_prob_lognormal.calculate_landslide_probability()
    np.testing.assert_almost_equal(
        grid_2.at_node['landslide__probability_of_failure'][5], 0.8)
    np.testing.assert_almost_equal(
        grid_2.at_node['landslide__probability_of_failure'][9], 0.4)

def test_non_euclidean_kneighbors():
    rng = np.random.RandomState(0)
    X = rng.rand(5, 5)

    # Find a reasonable radius.
    dist_array = pairwise_distances(X).flatten()
    np.sort(dist_array)
    radius = dist_array[15]

    # Test kneighbors_graph
    for metric in ['manhattan', 'chebyshev']:
        nbrs_graph = neighbors.kneighbors_graph(
            X, 3, metric=metric).toarray()
        nbrs1 = neighbors.NearestNeighbors(3, metric=metric).fit(X)
        assert_array_equal(nbrs_graph, nbrs1.kneighbors_graph(X).toarray())

    # Test radiusneighbors_graph
    for metric in ['manhattan', 'chebyshev']:
        nbrs_graph = neighbors.radius_neighbors_graph(
            X, radius, metric=metric).toarray()
        nbrs1 = neighbors.NearestNeighbors(metric=metric, radius=radius).fit(X)
        assert_array_equal(nbrs_graph,
                           nbrs1.radius_neighbors_graph(X).toarray())

    # Raise error when wrong parameters are supplied,
    X_nbrs = neighbors.NearestNeighbors(3, metric='manhattan')
    X_nbrs.fit(X)
    assert_raises(ValueError, neighbors.kneighbors_graph, X_nbrs, 3,
                  metric='euclidean')
    X_nbrs = neighbors.NearestNeighbors(radius=radius, metric='manhattan')
    X_nbrs.fit(X)
    assert_raises(ValueError, neighbors.radius_neighbors_graph, X_nbrs,
                  radius, metric='euclidean')

def unit_maker(func, func0):
    "Test bn.(arg)partsort gives same output as bn.slow.(arg)partsort."
    msg = '\nfunc %s | input %s (%s) | shape %s | n %d | axis %s\n'
    msg += '\nInput array:\n%s\n'
    for i, arr in enumerate(arrays()):
        for axis in list(range(-arr.ndim, arr.ndim)) + [None]:
            if axis is None:
                n = arr.size
            else:
                n = arr.shape[axis]
            n = max(n // 2, 1)
            with np.errstate(invalid='ignore'):
                actual = func(arr.copy(), n, axis=axis)
                actual[:n] = np.sort(actual[:n], axis=axis)
                actual[n:] = np.sort(actual[n:], axis=axis)
                desired = func0(arr.copy(), n, axis=axis)
                if 'arg' in func.__name__:
                    desired[:n] = np.sort(desired[:n], axis=axis)
                    desired[n:] = np.sort(desired[n:], axis=axis)
            tup = (func.__name__, 'a'+str(i), str(arr.dtype),
                   str(arr.shape), n, str(axis), arr)
            err_msg = msg % tup
            assert_array_equal(actual, desired, err_msg)
            err_msg += '\n dtype mismatch %s %s'
            if hasattr(actual, 'dtype') or hasattr(desired, 'dtype'):
                da = actual.dtype
                dd = desired.dtype
                assert_equal(da, dd, err_msg % (da, dd))

def index_trim_outlier(resid, k):
    '''returns indices to residual array with k outliers removed

    Parameters
    ----------
    resid : array_like, 1d
        data vector, usually residuals of a regression
    k : int
        number of outliers to remove

    Returns
    -------
    trimmed_index : array, 1d
        index array with k outliers removed
    outlier_index : array, 1d
        index array of k outliers

    Notes
    -----

    Outliers are defined as the k observations with the largest
    absolute values.

    '''
    sort_index = np.argsort(np.abs(resid))
    # index of non-outlier
    trimmed_index = np.sort(sort_index[:-k])
    outlier_index = np.sort(sort_index[-k:])
    return trimmed_index, outlier_index

def pick_channels(ch_names, include, exclude=[]):
    """Pick channels by names

    Returns the indices of the good channels in ch_names.

    Parameters
    ----------
    ch_names : list of string
        List of channels.
    include : list of string
        List of channels to include (if empty include all available).
    exclude : list of string
        List of channels to exclude (if empty do not exclude any channel).
        Defaults to [].

    Returns
    -------
    sel : array of int
        Indices of good channels.
    """
    if len(np.unique(ch_names)) != len(ch_names):
        raise RuntimeError('ch_names is not a unique list, picking is unsafe')
    sel = []
    for k, name in enumerate(ch_names):
        if (len(include) == 0 or name in include) and name not in exclude:
            sel.append(k)
    sel = np.unique(sel)
    np.sort(sel)
    return sel

	def check_obs_scheme(self):
		" Checks the internal validity of provided observation schemes "

		# check sub_pops
		idx_union = np.sort(self._sub_pops[0])
		i = 1
		while idx_union.size < self._p and i < len(self._sub_pops):
			idx_union = np.union1d(idx_union, self._sub_pops[i]) 
			i += 1
		if idx_union.size != self._p or np.any(idx_union!=np.arange(self._p)):
			raise Exception(('all subpopulations together have to cover '
			'exactly all included observed varibles y_i in y.'
			'This is not the case. Change the difinition of '
			'subpopulations in variable sub_pops or reduce '
			'the number of observed variables p. '
			'The union of indices of all subpopulations is'),
			idx_union )

		# check obs_time
		if not self._obs_time[-1]==self._T:
			raise Exception(('Entries of obs_time give the respective ends of '
							'the periods of observation for any '
							'subpopulation. Hence the last entry of obs_time '
							'has to be the full recording length. The last '
							'entry of obs_time before is '), self._obs_time[-1])

		if np.any(np.diff(self._obs_time)<1):
			raise Exception(('lengths of observation have to be at least 1. '
							'Minimal observation time for a subpopulation: '),
							np.min(np.diff(self._obs_time)))

		# check obs_pops
		if not self._obs_time.size == self._obs_pops.size:
			raise Exception(('each entry of obs_pops gives the index of the '
							'subpopulation observed up to the respective '
							'time given in obs_time. Thus the sizes of the '
							'two arrays have to match. They do not. '
							'no. of subpop. switch points and no. of '
							'subpopulations ovserved up to switch points '
							'are '), (self._obs_time.size, self._obs_pops.size))

		idx_pops = np.sort(np.unique(self._obs_pops))
		if not np.min(idx_pops)==0:
			raise Exception(('first subpopulation has to have index 0, but '
							'is given the index '), np.min(idx_pops))
		elif not idx_pops.size == len(self._sub_pops):
			raise Exception(('number of specified subpopulations in variable '
							'sub_pops does not meet the number of '
							'subpopulations indexed in variable obs_pops. '
							'Delete subpopulations that are never observed, '
							'or change the observed subpopulations in '
							'variable obs_pops accordingly. The number of '
							'indexed subpopulations is '),
							len(self._sub_pops))
		elif not np.all(np.diff(idx_pops)==1):
			raise Exception(('subpopulation indices have to be consecutive '
							'integers from 0 to the total number of '
							'subpopulations. This is not the case. '
							'Given subpopulation indices are '),
							idx_pops)

 def test_fit(self):
   self.kmeans.fit(input_fn=self.input_fn(), steps=10)
   centers = normalize(self.kmeans.clusters())
   self.assertAllClose(
       np.sort(
           centers, axis=0), np.sort(
               self.true_centers, axis=0))

def test_fetch_rcv1():
    try:
        data1 = fetch_rcv1(shuffle=False, download_if_missing=False)
    except IOError as e:
        if e.errno == errno.ENOENT:
            raise SkipTest("Download RCV1 dataset to run this test.")

    X1, Y1 = data1.data, data1.target
    cat_list, s1 = data1.target_names.tolist(), data1.sample_id

    # test sparsity
    assert_true(sp.issparse(X1))
    assert_true(sp.issparse(Y1))
    assert_equal(60915113, X1.data.size)
    assert_equal(2606875, Y1.data.size)

    # test shapes
    assert_equal((804414, 47236), X1.shape)
    assert_equal((804414, 103), Y1.shape)
    assert_equal((804414,), s1.shape)
    assert_equal(103, len(cat_list))

    # test ordering of categories
    first_categories = [u'C11', u'C12', u'C13', u'C14', u'C15', u'C151']
    assert_array_equal(first_categories, cat_list[:6])

    # test number of sample for some categories
    some_categories = ('GMIL', 'E143', 'CCAT')
    number_non_zero_in_cat = (5, 1206, 381327)
    for num, cat in zip(number_non_zero_in_cat, some_categories):
        j = cat_list.index(cat)
        assert_equal(num, Y1[:, j].data.size)

    # test shuffling and subset
    data2 = fetch_rcv1(shuffle=True, subset='train', random_state=77,
                       download_if_missing=False)
    X2, Y2 = data2.data, data2.target
    s2 = data2.sample_id

    # test return_X_y option
    fetch_func = partial(fetch_rcv1, shuffle=False, subset='train',
                         download_if_missing=False)
    check_return_X_y(data2, fetch_func)

    # The first 23149 samples are the training samples
    assert_array_equal(np.sort(s1[:23149]), np.sort(s2))

    # test some precise values
    some_sample_ids = (2286, 3274, 14042)
    for sample_id in some_sample_ids:
        idx1 = s1.tolist().index(sample_id)
        idx2 = s2.tolist().index(sample_id)

        feature_values_1 = X1[idx1, :].toarray()
        feature_values_2 = X2[idx2, :].toarray()
        assert_almost_equal(feature_values_1, feature_values_2)

        target_values_1 = Y1[idx1, :].toarray()
        target_values_2 = Y2[idx2, :].toarray()
        assert_almost_equal(target_values_1, target_values_2)

def get_fn(data, fp):
    """ Given some scores data and a false negatives rate
    find the corresponding false positive rate in the ROC curve.
    If the point does not exist, we will interpolate it.

    """
    if fp in data.fpr:
        pos = np.where(data.fpr == fp)
        fnr, thr = np.mean(data.fnr[pos]), np.mean(data.thrs[pos])
    else:
        # Set data for interpolation
        x = np.sort(data.fpr)
        # Set new arange whichs includes the wanted value
        xnew = np.arange(fp, x[-1])
        # Interpolate the FN
        y = np.sort(data.tpr)
        f = interpolate.interp1d(x, y)
        tpr = f(xnew)[0]
        fnr = 1 - tpr
        # Interpolate the threashold
        y = np.sort(data.thrs)
        f = interpolate.interp1d(x, y)
        thr = f(xnew)[0]
    print("Dado el valor de fp: {0}, el valor de fnr es: {1} y el umbral: {2} "
          .format(fp, fnr, thr))

    def computeError(self, Res, method="None"):
        """ Compute median absolute and relative errors """
        absErr = np.abs(Res - self.trueRes)
        idx_nonzero = np.where(self.trueRes != 0)
        absErr_nonzero = absErr[idx_nonzero]
        true_nonzero = self.trueRes[idx_nonzero]
        relErr = absErr_nonzero / true_nonzero

        # log_str_rel = "\n".join(map(str, relErr))
        # log_str_abs = "\n".join(map(str, absErr))

        if Params.IS_LOGGING:
            log_str = ""
            for i in range(len(self.query_list)):
                area = rect_area(self.query_list[i])
                query_str = str(self.query_list[i][0][0]) + "\t" + str(self.query_list[i][0][1]) + "\t" + str(
                    self.query_list[i][1][0]) + "\t" + str(self.query_list[i][1][1]) + "\t" + str(area)
                err_str = str(self.trueRes[i]) + "\t" + str(Res[i]) + "\t" + str(absErr[i]) + "\t" + str(relErr[i])
                log_str = log_str + query_str + "\t" + err_str + "\n"
            log(method, log_str)

        absErr = np.sort(absErr)
        relErr = np.sort(relErr)
        n_abs = len(absErr)
        n_rel = len(relErr)
        return absErr[int(n_abs / 2)], relErr[int(n_rel / 2)]

def stats(arr):
    """ Show the minimum, maximum median, mean, shape and size of an
    array.

    Also show the number of NaN entries (if any).
    """
    arr = np.asarray(arr)
    shape = arr.shape
    arr = arr.ravel()
    size = len(arr)
    bad = np.isnan(arr)
    nbad = bad.sum()
    if nbad == size:
        return "#NaN %i of %i" % (nbad, size)
    elif nbad == 0:
        arr = np.sort(arr)
    else:
        arr = np.sort(arr[~bad])
    if len(arr) % 2 == 0:
        i = len(arr) // 2
        median = 0.5 * (arr[i - 1] + arr[i])
    else:
        median = arr[len(arr) // 2]

    return "min %.5g max %.5g median %.5g mean %.5g shape %s #NaN %i of %i" % (
        arr[0],
        arr[-1],
        median,
        arr.mean(),
        shape,
        nbad,
        size,
    )