def quality_fis(self,fis):
        """
        Count the correct classifications of a given FIS on the check data.

        :param fis: The Fuzzy Inference System to be tested
        :type fis: :class:`rule.ClassifierSet`
        :returns: A tuple containing the number of correct classifications and the total number of classifications
        :rtype: (:class:`int`,:class:`int`)
        """
        if fis.dimension() == self.training_data.shape[1]:
            last_res = 0.0
            count = 0
            for i in range(self.check_data.shape[0]):
                last_res = fis.evaluate(np.hstack((self.check_data[i],last_res)))
                if abs(last_res - self.id) < 0.5:
                    count = count + 1
            return (count,self.check_data.shape[0])
        else:
            rvec = fis.evaluates(self.check_data) - self.id
            rvec = ma.masked_inside(rvec,-0.5,0.5)
            return (ma.count_masked(rvec),self.check_data.shape[0])
        
        if fis.dimension() == self.training_data.shape[1]:
            dat = np.hstack((self.check_data,self.id*np.ones((self.check_data.shape[0],1))))
        else:
            dat = self.check_data
        #if self.check_data.shape[1] == self.training_data.shape[1]:
        #    dat = self.check_data
        #else:
        #    dat = np.hstack((self.check_data,np.zeros((self.check_data.shape[0],1))))
        rvec = fis.evaluates(dat) - self.id
        rvec = ma.masked_inside(rvec,-0.5,0.5)
        return (ma.count_masked(rvec),self.check_data.shape[0])

 def action(self, Data) :
     params = self.params
     # Keep track of how many pre existing flags there are for feedback
     # purposes.
     already_flagged = ma.count_masked(Data.data)
     if params["rotate"]:
         if (tuple(Data.field['CRVAL4']) == (1, 2, 3, 4)):
             rotate_pol.rotate(Data, (-5,-7,-8,-6))
             Data.add_history('Rotated to XX,XY,YX,YY')
     # Few operations to be performed before flagging.
     if params["perform_hanning"] :
         hanning.hanning_smooth(Data)
         Data.add_history('Hanning smoothed.')
     if params["cal_scale"] :
         cal_scale.scale_by_cal(Data, True, False, False)
         Data.add_history('Converted to units of noise cal temperture.')
     # Flag the data.
     apply_cuts(Data, sigma_thres=params['sigma_thres'], 
                 badness_thres=params['badness_thres'],
                 time_cut=params['time_cut'])
     Data.add_history('Flagged Bad Data.', ('Sigma threshold: '
                 + str(self.params['sigma_thres']), 'Badness threshold: '
                 + str(self.params['badness_thres']), 'Time mask size: '
                 + str(self.params['time_cut'])))
     # Report the number of new flags.
     new_flags = ma.count_masked(Data.data) - already_flagged
     self.block_feedback = str(new_flags) + ', '
     return Data

 def process_file(self, file_ind) :
     params = self.params
     file_middle = params['file_middles'][file_ind]
     input_fname = (params['input_root'] + file_middle +
                    params['input_end'])
     sub_input_fname = (params['subtracted_input_root'] + file_middle
                        + params['input_end'])
     output_fname = (params['output_root']
                     + file_middle + params['output_end'])
     sub_output_fname = (params['subtracted_output_root']
                         + file_middle + params['output_end'])
     Writer = fitsGBT.Writer(feedback=self.feedback)
     SubWriter = fitsGBT.Writer(feedback=self.feedback)
     
     # Read in the data, and loop over data blocks.
     Reader = fitsGBT.Reader(input_fname, feedback=self.feedback)
     SubReader = fitsGBT.Reader(sub_input_fname, feedback=self.feedback)
     if (sp.any(Reader.scan_set != SubReader.scan_set)
         or sp.any(Reader.IF_set != SubReader.IF_set)) :
         raise ce.DataError("IFs and scans don't match signal subtracted"
                            " data.")
     # Get the number of scans if asked for all of them.
     scan_inds = params['scans']
     if len(scan_inds) == 0 or scan_inds is None :
         scan_inds = range(len(Reader.scan_set))
     if_inds = params['IFs']
     if len(if_inds) == 0 or scan_inds is None :
         if_inds = range(len(Reader.IF_set))
     if self.feedback > 1 :
         print "New flags each block:",
     # Loop over scans and IFs
     for thisscan in scan_inds :
         for thisIF in if_inds :
             Data = Reader.read(thisscan, thisIF)
             SubData = SubReader.read(thisscan, thisIF)
             n_flags = ma.count_masked(Data.data)
             # Now do the flagging.
             flag(Data, SubData, params['thres'])
             Data.add_history("Reflaged for outliers.", ("Used file: "
                 + utils.abbreviate_file_path(sub_input_fname),))
             SubData.add_history("Reflaged for outliers.")
             Writer.add_data(Data)
             SubWriter.add_data(SubData)
             # Report the numbe of new flags.
             n_flags = ma.count_masked(Data.data) - n_flags
             if self.feedback > 1 :
                 print n_flags,
     if self.feedback > 1 :
         print ''
     # Finally write the data back to file.
     utils.mkparents(output_fname)
     utils.mkparents(sub_output_fname)
     Writer.write(output_fname)
     SubWriter.write(sub_output_fname)

 def test_flags(self) :
     self.Data.data[...] = 10
     self.Data.data[4, 0, 0, 235] = 100
     nf = self.Data.dims[-1]
     self.DataSub.data[0::2, :, :, :] = sp.arange(1, nf+1)
     self.DataSub.data[1::2, :, :, :] = 0
     self.DataSub.data[3, 0, 0, 458] = 2*458
     self.DataSub.data[3, 0, 0, 986] = 2*986
     self.DataSub.data[8, 0, 1, 734] = 2*734
     reflag.flag(self.Data, self.DataSub, thres=2.0)
     self.assertTrue(self.Data.data[3, 0, 0, 458] is ma.masked)
     self.assertTrue(self.Data.data[3, 0, 0, 986] is ma.masked)
     self.assertTrue(self.Data.data[8, 0, 1, 734] is ma.masked)
     self.assertEqual(ma.count_masked(self.Data.data), 3)
     self.assertEqual(ma.count_masked(self.DataSub.data), 3)

 def register(self):
     """
     We have all the sie and timestamp values in dictionaries, so just
     build the time step sequence from them
     """
     self.numSteps = len(self.StepDict)
     self.Steps = ma.empty(self.numSteps, dtype=np.int32)
     self.Steps.mask = True
     self.TS_IDs = ma.empty(self.numSteps, dtype=np.int32)
     self.TS_IDs.mask = True
     self.Steps.soften_mask()
     self.TS_IDs.soften_mask()
     for sie,index in self.StepDict.iteritems():
         self.Steps[index] = sie
         self.TS_IDs[index] = self.TS_IDDict[sie]
     self.__registered += 1
     if ma.count_masked(self.Steps) != 0:
         handleError(self,
                     TimeStepsRegisterError,
                     "TimeSteps.register(): Warning - registered all %d, but still have %d masked values" % (self.numSteps, len(np.nonzero(self.Steps.mask))))
         # not reached
     if self.Debug == True:
         self.DebugMessages += "TimeSteps.register(): Registred all %d expected values, now to check them and take the Step differences" % self.numSteps
     self.Diff = np.diff(self.Steps)
     if (np.min(self.Diff) <= 0) and (self.Debug == True):
         message = "TimeSteps.register(): Warning - negative or zero time differentials at\n"
         message += "indices:" + str(np.where(self.Diff <= 0)) + "values:" + str(self.Diff[np.where(self.Diff <= 0)])
         handleError(self,
                     TimeStepsRegisterError,
                     message)
         # not reached
     self.HaveData = True

 def interpolate(self):
     # N.B. The counter interpolate() fucntion also divides by
     # the step intervals, so the resulting series is a set of
     # observations of the true rate in MB/s.
     masked_vals = np.where(self.Read.Values.mask == True)
     self.Missing = np.zeros(len(self.Read.Values), dtype=np.int32)
     if ma.count_masked(self.Read.Values) != 0:
         self.Missing[masked_vals] = 1
     n = self.Read.interpolate()
     if n == 0:
         handleError(self,
                     OSTNoDataError,
                     "OST.interpolate(): Warning - No Read data")
         # not reached
     n = self.Write.interpolate()
     if n == 0:
         handleError(self,
                     OSTNoDataError,
                     "OST.interpolate(): Warning - No Write data")
         # not reached
     n = self.OST.interpolate()
     if n == 0:
         handleError(self,
                     OSTNoDataError,
                     "OST.interpolate(): Warning - No combined data")

    def test_slicing(self):
        cube = self._load_3d_cube()

        # Test the slicing before deferred loading
        full_slice = cube[3]
        partial_slice = cube[0]
        self.assertIsInstance(full_slice.data, np.ndarray)
        self.assertIsInstance(partial_slice.data, ma.core.MaskedArray)
        self.assertEqual(ma.count_masked(partial_slice.data), 25)

        # Test the slicing is consistent after deferred loading
        full_slice = cube[3]
        partial_slice = cube[0]
        self.assertIsInstance(full_slice.data, np.ndarray)
        self.assertIsInstance(partial_slice.data, ma.core.MaskedArray)
        self.assertEqual(ma.count_masked(partial_slice.data), 25)

 def register(self):
     """
     We have all the bin and timestamp values in dictionaries, so just
     build the time bin sequence from them
     """
     self.numBins = len(self.BinDict)
     self.Bins = ma.empty(self.numBins, dtype=np.float64)
     self.Bins.mask = True
     self.Bins.soften_mask()
     # the bins some times emerge in a random order
     # so put them in order, and then preserve that
     count = 0
     for bin,index in sorted(self.BinDict.iteritems()):
         self.Bins[count] = bin
         count += 1
     for count in range(self.numBins):
         self.BinDict[self.Bins[count]] = count
     self.__registered += 1
     if ma.count_masked(self.Bins) != 0:
         handleError(self,
                     HistBinsRegisterError,
                     "HistBins.register(): Warning - registered all %d, but still have %d masked values" % (self.numBins, len(np.nonzero(self.Bins.mask))))
         # not reached
     if self.Debug == True:
         self.DebugMessages += "HistBins.register(): Registred all %d expected values, now to check them and take the Bin differences" % self.numBins
     self.HaveData = True

 def check_for_all(self):
     """
     Function to check if all the values are masked or all not masked
     """
     masked_N = ma.count_masked(self.xs, axis=1)[np.newaxis,:]
     is_all_masked = self.N == masked_N
     is_none_masked = masked_N == 0
     return is_none_masked + is_all_masked

 def interpolate(self):
     # If there are no masked values there is nothing to do
     if ma.count_masked(self.Values) == 0:
         if self.Debug == True:
             self.DebugMessages += "Series2.interpolate(): No masked values"
         return(len(self.Values))
     for vIndex in range(self.width):
         self.interpolateItem(vIndex)

    def test_slicing(self):
        cube = self._load_3d_cube()

        # Test the slicing before deferred loading
        full_slice = cube[3]
        partial_slice = cube[0]
        self.assertTrue(isinstance(full_slice.data, np.ndarray), "Expected a numpy array")
        self.assertTrue(isinstance(partial_slice.data, ma.core.MaskedArray), "Expected a numpy.ma.core.MaskedArray")
        self.assertEqual(ma.count_masked(partial_slice._data), 25)

        # Test the slicing is consistent after deferred loading
        cube.data
        full_slice = cube[3]
        partial_slice = cube[0]
        self.assertTrue(isinstance(full_slice.data, np.ndarray), "Expected a numpy array")
        self.assertTrue(isinstance(partial_slice.data, ma.core.MaskedArray), "Expected a numpy.ma.core.MaskedArray")
        self.assertEqual(ma.count_masked(partial_slice._data), 25)

 def test_scalar_with_overlap_above_mdtol(self):
     # Slice src so result collapses to a scalar.
     src_cube = self.src_cube[:, 1, :]
     # Regrid to a single cell with 50% overlap with masked src cells.
     grid_cube = self.grid_cube[3, 1, 4]
     # Set threshold (mdtol) to less than 0.5 (50%).
     res = regrid(src_cube, grid_cube, mdtol=0.4)
     self.assertEqual(ma.count_masked(res.data), 1)

    def action(self, Data):
        '''Prepares Data and flags RFI.
        
        Parameters
        ----------
        Data : DataBlock
            Contains information in a usable format direct from GBT. 

        Returns
        -------
        Data : DataBlock
            The input `Data` with RFI flagged. Will also be cal scaled and
            rotated to XX,YY... if so chosen.

        '''
        params = self.params
        # Keep track of how many pre existing flags there are for feedback
        # purposes.
        already_flagged = ma.count_masked(Data.data)
        if params["rotate"]:
            if (tuple(Data.field['CRVAL4']) == (1, 2, 3, 4)):
                rotate_pol.rotate(Data, (-5,-7,-8,-6))
                Data.add_history('Rotated to XX,XY,YX,YY')
        # Few operations to be performed before flagging.
        if params["perform_hanning"] :
            hanning.hanning_smooth(Data)
            Data.add_history('Hanning smoothed.')
        if params["cal_scale"] or params["cal_phase"]:
            cal_scale.scale_by_cal(Data, params['cal_scale'], False, False,
                                  False, rotate=params['cal_phase'])
            Data.add_history('Converted to units of noise cal temperture.')
        # Flag the data.
        apply_cuts(Data, sigma_thres=params['sigma_thres'], 
                    badness_thres=params['badness_thres'],
                    time_cut=params['time_cut'], 
                    submean=params['submean'])
        Data.add_history('Flagged Bad Data.', ('Sigma threshold: '
                    + str(self.params['sigma_thres']), 'Badness threshold: '
                    + str(self.params['badness_thres']), 'Time mask size: '
                    + str(self.params['time_cut'])))
        # Report the number of new flags.
        new_flags = ma.count_masked(Data.data) - already_flagged
        percent = float(new_flags) / Data.data.size * 100
        self.block_feedback = '%d (%f%%), ' % (new_flags, percent)
        return Data

def monotonic(array, strict=False, return_direction=False):
    """
    Return whether the given 1d array is monotonic.

    Note that, the array must not contain missing data.

    Kwargs:

    * strict (boolean)
        Flag to enable strict monotonic checking
    * return_direction (boolean)
        Flag to change return behaviour to return
        (monotonic_status, direction). Direction will be 1 for positive
        or -1 for negative. The direction is meaningless if the array is
        not monotonic.

    Returns:

    * monotonic_status (boolean)
        Whether the array was monotonic.

        If the return_direction flag was given then the returned value
        will be:

            ``(monotonic_status, direction)``

    """
    if array.ndim != 1 or len(array) <= 1:
        raise ValueError('The array to check must be 1 dimensional and have '
                         'more than 1 element.')

    if ma.isMaskedArray(array) and ma.count_masked(array) != 0:
        raise ValueError('The array to check contains missing data.')

    # Identify the directions of the largest/most-positive and
    # smallest/most-negative steps.
    d = np.diff(array)

    sign_max_d = np.sign(np.max(d))
    sign_min_d = np.sign(np.min(d))

    if strict:
        monotonic = sign_max_d == sign_min_d and sign_max_d != 0
    else:
        monotonic = (sign_min_d < 0 and sign_max_d <= 0) or \
                    (sign_max_d > 0 and sign_min_d >= 0) or \
                    (sign_min_d == sign_max_d == 0)

    if return_direction:
        if sign_max_d == 0:
            direction = sign_min_d
        else:
            direction = sign_max_d

        return monotonic, direction

    return monotonic

    def params_to_array_zeromask(self, params):
        """
        Returns tuple of arrays + list
        Process params input into appropriate arrays by setting them to
        zero if param in params in zero and removing them from params,
        otherwise they stay in params and value remains masked

        Parameters
        ----------
        params : list
            list of values to fill in masked values

        Returns
        -------
        lam_0 : numpy array
        lam_1 : numpy array
        delta_0 : numpy array
        delta_1 : numpy array
        mu : numpy array
        phi : numpy array
        sigma : numpy array
        guesses : list
            List of remaining params after filtering and filling those
            that were zero
        """
        paramcopy = params[:]
        lam_0_e = self.lam_0_e.copy()
        lam_1_e = self.lam_1_e.copy()
        delta_0_e = self.delta_0_e.copy()
        delta_1_e = self.delta_1_e.copy()
        mu_e = self.mu_e.copy()
        phi_e = self.phi_e.copy()
        sigma_e = self.sigma_e.copy()

        all_arrays = [lam_0_e, lam_1_e, delta_0_e, delta_1_e, mu_e, phi_e,
                      sigma_e]

        arg_sep = self._gen_arg_sep([ma.count_masked(struct) for struct in \
                                     all_arrays])

        guesses = []
        # check if each element is masked or not
        for struct in all_arrays:
            it = np.nditer(struct.mask, flags=['multi_index'])
            while not it.finished:
                if it[0]:
                    val = paramcopy.pop(0)
                    if val == 0:
                        struct[it.multi_index] = 0
                    else:
                        guesses.append(val)
                it.iternext()

        return tuple(all_arrays + [guesses])

def printstuff(vals,vals_sum,file_count):
    # create running sum of values
    print file_count
    print '----------------'
    print 'sum', ma.sum(vals)
    print 'total sum', ma.sum(vals_sum)
    print 'perc masked:', ma.count_masked(vals_sum)/vals_sum.size*100.,'%'
    print '----------------'
    print 'max', ma.amax(vals)
    print 'min', ma.amin(vals)
    print '\n----------------'

    def format_and_clean_data_main(self):
        """
        Main function to format and clean data based on choices by the user.
        """
        # Check if over missing_bound percent or missing_bound number of values are missing
        too_many_missing = self.has_too_many_missing(self.init_perc_remove)
        if ma.any(too_many_missing):
            idx, = ma.where(too_many_missing)
            self.xs[idx] = ma.mask_rows(self.xs[idx])

        # Check array to see if it is filled with values or empty
        if ma.all(self.check_for_all()):
            return self.xs

        # Clean outliers
        self.clean_outliers()

        # Take average of neighbor values to fill up to a given missing value gap length
        self.clean_gaps_w_linspace(fill_gap_length=self.max_gap_length)
        if ma.all(ma.count_masked(self.xs[:, :-self.keep_n_values], axis=1)[np.newaxis,:] == 0):
            return self.xs # if no masked values remain in values before recent ones

        # Remove values if they start the array and are then followed by too many masked values
        start_idx = self.find_new_starting_value()

        # If there are over x% blank values left in the original data after above changes,
        # check to see if x% of the blanks fall after the new start year
        too_many_missing = self.has_too_many_missing(self.second_perc_remove) # boolean array
        if ma.any(too_many_missing):
            n_masked = np.array([ma.count_masked(self.xs[i,s_idx:])
                                 for i, s_idx in enumerate(start_idx)]) / self.N > self.perc_remove_after_start_idx
            if ma.any(n_masked):
                idx, = ma.where(n_masked)
                self.xs[idx] = ma.mask_rows(self.xs[idx])

        # To fill in remaining values, run linear regression on non-zero values
        self.clean_gaps_w_lin_regress(start_idx)

        # If linear regression left negative or zero values, then use linear space to fill in middle gaps
        if ma.any(ma.masked_less_equal(self.xs, 0.)):
            self.clean_gaps_w_linspace()

 def test_002_flagging_cut(self):
     ind = (6,1,1,676)
     freq = 676
     ind2 = (3,2,0,245)
     freq2 = 245
     self.Data.data[ind] += 0.2
     self.Data.data[ind2] += 0.2
     flag_data.apply_cuts(self.Data)
     self.assertFalse(False in self.Data.data[:,:,:,freq].mask)
     self.assertFalse(False in self.Data.data[:,:,:,freq2].mask)
     self.assertTrue(float(ma.count_masked(self.Data.data)) / 
                     float(self.Data.data.size) < 0.1)

 def test_rebinned_cal(self) :
     rebin_freq.rebin(self.CalData, 1.0)
     DataCopy = copy.deepcopy(self.Data)
     DataCopy.calc_freq()
     calibrate.multiply_by_cal(self.Data, self.CalData)
     gains = sp.array([1,sp.sqrt(5),sp.sqrt(5),5])
     gains.shape = (1,4,1,1)
     expected = (DataCopy.data*gains*DataCopy.freq)
     self.assertTrue(ma.count_masked(self.Data.data <
                        sp.size(self.Data.data)*50.0/2000.0))
     expected.mask = sp.array(self.Data.data.mask)
     self.assertTrue(ma.allclose(expected, self.Data.data, 4))

    def test_params_to_array_zeromask(self):
        """
        Tests if params_to_array_zeromask function works correctly. In order to
        pass, params_to_array_zeromask must return masked arrays with the
        guess_params elements that are zero unmasked and set to zero in the
        appropriate arrays. The new guess_params array is also returned with
        those that were 0 removed. If both of these are not returned correctly,
        the test fails.
        """
        guess_params_arr = np.array(self.guess_params)
        neqs = self.affine_obj.neqs
        guess_params_arr[:neqs] = 0
        guess_params = guess_params_arr.tolist()
        guess_length = self.affine_obj._gen_guess_length()
        params_guesses = self.affine_obj.params_to_array_zeromask(guess_params)
        updated_guesses = params_guesses[-1]
        self.assertEqual(len(updated_guesses), len(guess_params) - neqs)

        # ensure that number of masked has correctly been set
        count_masked_new = ma.count_masked(params_guesses[0])
        count_masked_orig = ma.count_masked(self.affine_obj.lam_0_e)
        self.assertEqual(count_masked_new, count_masked_orig - neqs)