def register(self):
     """
     We have all the sie and timestamp values in dictionaries, so just
     build the time step sequence from them
     """
     self.numSteps = len(self.StepDict)
     self.Steps = ma.empty(self.numSteps, dtype=np.int32)
     self.Steps.mask = True
     self.TS_IDs = ma.empty(self.numSteps, dtype=np.int32)
     self.TS_IDs.mask = True
     self.Steps.soften_mask()
     self.TS_IDs.soften_mask()
     for sie,index in self.StepDict.iteritems():
         self.Steps[index] = sie
         self.TS_IDs[index] = self.TS_IDDict[sie]
     self.__registered += 1
     if ma.count_masked(self.Steps) != 0:
         handleError(self,
                     TimeStepsRegisterError,
                     "TimeSteps.register(): Warning - registered all %d, but still have %d masked values" % (self.numSteps, len(np.nonzero(self.Steps.mask))))
         # not reached
     if self.Debug == True:
         self.DebugMessages += "TimeSteps.register(): Registred all %d expected values, now to check them and take the Step differences" % self.numSteps
     self.Diff = np.diff(self.Steps)
     if (np.min(self.Diff) <= 0) and (self.Debug == True):
         message = "TimeSteps.register(): Warning - negative or zero time differentials at\n"
         message += "indices:" + str(np.where(self.Diff <= 0)) + "values:" + str(self.Diff[np.where(self.Diff <= 0)])
         handleError(self,
                     TimeStepsRegisterError,
                     message)
         # not reached
     self.HaveData = True

def to_ham6(img, palette, background=None, out=None):
    _debug_array(img)

    if background is None:
        background = ma.masked
    elif isinstance(background, numbers.Integral):
        background = palette[background]

    if not ma.is_masked(background) and ma.isMaskedArray(img):
        img = img.filled(background)

    if ma.isMaskedArray(img):
        ham6 = ma.empty(img.shape[:2], dtype=np.uint8)
    else:
        ham6 = np.empty(img.shape[:2], dtype=np.uint8)

    for y in range(img.shape[0]):
        c = background
        for x in range(img.shape[1]):
            i, c = ham6_nearest(img[y, x], palette, c)
            ham6[y, x] = i
            if out is not None:
                out[y, x] = c

    _debug_array(ham6)
    return ham6

def XWrap(x,ifold,fill_value=0):
    """
    Extend and wrap array.
    
    Fold array every y indecies.  There will typically be a hanging
    part of the array.  This is padded out.

    Parameters
    ----------

    x     : input
    ifold : Wrap array after ifold indecies.

    Return
    ------

    xwrap : Wrapped array.

    """

    ncad = x.size # Number of cadences
    nrow = int(np.floor(ncad/ifold) + 1)
    nExtend = nrow * ifold - ncad # Pad out remainder of array with 0s.

    if type(x) is np.ma.core.MaskedArray:
        pad = ma.empty(nExtend)
        pad.mask = True
        x = ma.hstack( (x ,pad) )
    else:    
        pad = np.empty(nExtend) 
        pad[:] = fill_value
        x = np.hstack( (x ,pad) )
    xwrap = x.reshape( nrow,-1 )

    return xwrap

def from_ham6(ham6, palette, background=None):
    if background is None:
        background = ma.masked
    elif isinstance(background, numbers.Integral):
        background = palette[background]

    if ma.is_masked(background) or ma.isMaskedArray(ham6):
        rgb8 = ma.empty(ham6.shape[:2] + (3,), dtype=np.uint8)
    else:
        rgb8 = np.empty(ham6.shape[:2] + (3,), dtype=np.uint8)

    for y in range(rgb8.shape[0]):
        c = background
        for x in range(rgb8.shape[1]):
            i = ham6[y, x]
            if i is ma.masked:
                ham6[y, x] = ma.masked
                continue

            if i < 0x10:
                c = palette[i]
            else:
                c = c.copy()
                c[(None, 2, 0, 1)[i >> 4]] = (i & 0xF) * 0x11

            rgb8[y, x] = c

    return rgb8

 def masked_array(self):
     data = ma.empty(self.shape, dtype=self.dtype,
                     fill_value=self.fill_value)
     for index in np.ndindex(self._stack.shape):
         masked_array = self._stack[index].masked_array()
         data[index] = masked_array
     return data

def _getdatafromsql(connection, tmp_table, query):
    """
    Private function creating a ndarray from the current table.

    Parameters
    ----------
    connection: sqlite3.Connection
        Current SQL connection.
    tmp_table: string
        Name of the temporary table created for the purpose of keeping ids when WHERE is used
    query: string
        SQL query.
    """
    # Transforms the typestr into dtypes
    
    # Define and execute the query
    connection.execute("CREATE TEMPORARY TABLE %s AS %s"%(tmp_table, query))
    
    # Get the list of names and types from the pragma
    pragmastr = "PRAGMA TABLE_INFO(%s)"%tmp_table
    (names, typestr) = zip(*(_[1:3] for _ in connection.execute(pragmastr).fetchall()))
    ndtype = []
    for (i, (n, t)) in enumerate(zip(names, typestr)):
        
        # Transform the name into a regular string (not unicode)
        n = str(n)
        if t =='INTEGER':
            ndtype.append((n, int))
        elif t =='TEXT':
            ndtype.append((n, '|S30'))
        elif t == 'BLOB':
            ndtype.append((n, object))
        else:
            ndtype.append((n, float))
    
    # Construct the ndarray
    connection.row_factory = sqlite3.Row
    data = connection.execute("SELECT * FROM %s"%tmp_table).fetchall()
    try:
        return np.array(data, dtype=ndtype)
    except TypeError:
        output = ma.empty(len(data), dtype=ndtype)
        
        # Find the index of the first row (0 or 1)?
        rowidref = connection.execute("SELECT rowid FROM %s LIMIT 1"%tmp_table).fetchone()[0]
        
        # Loop through the different fields identifying the null fields to mask
        maskstr_template = "SELECT rowid FROM %s WHERE %%s IS NULL"%tmp_table
        datastr_template = "SELECT %%s FROM %s WHERE %%s IS NOT NULL"%tmp_table
        for (i, field) in enumerate(names):
            current_output = output[field]
            current_mask = current_output._mask
            maskstr = maskstr_template % field
            maskidx = [_[0] - rowidref for _ in connection.execute(maskstr).fetchall()]
            current_mask[maskidx] = True
            datastr = datastr_template % (field, field)
            np.place(current_output._data, ~current_mask,
                [_[0] for _ in connection.execute(datastr).fetchall()])
        connection.execute("DROP TABLE %s"%tmp_table)
        return output

    def execute(self) :
        
        params = self.params
        n_files = len(params['file_middles'])
        scan_number = 0

        for file_middle in params['file_middles'] :
            file_name = (params['input_root'] + file_middle + 
                         params['input_end'])
            f = open(file_name, 'r')
            scan_list = cPickle.load(f)
            n_scans_file = len(scan_list)
            for scan in scan_list :
                if scan_number == 0 :
                    n_scans = n_scans_file*n_files
                    gain = ma.empty((n_scans,) + scan['gain'].shape)
                    time_arr = sp.empty(n_scans, dtype=int)
                gain[scan_number, ...] = scan['gain']
                tstring = str(scan['time']).split('.')[0]
                to = time.strptime(tstring, "%Y-%m-%dT%H:%M:%S")
                time_arr[scan_number] = to.tm_sec + 60*(to.tm_min + 
                    60*(to.tm_hour + 24*(to.tm_yday + 365*(to.tm_year-2000))))
                scan_number += 1
        self.time = time_arr[:scan_number]
        self.gain = gain[:scan_number,...]

 def register(self):
     """
     We have all the bin and timestamp values in dictionaries, so just
     build the time bin sequence from them
     """
     self.numBins = len(self.BinDict)
     self.Bins = ma.empty(self.numBins, dtype=np.float64)
     self.Bins.mask = True
     self.Bins.soften_mask()
     # the bins some times emerge in a random order
     # so put them in order, and then preserve that
     count = 0
     for bin,index in sorted(self.BinDict.iteritems()):
         self.Bins[count] = bin
         count += 1
     for count in range(self.numBins):
         self.BinDict[self.Bins[count]] = count
     self.__registered += 1
     if ma.count_masked(self.Bins) != 0:
         handleError(self,
                     HistBinsRegisterError,
                     "HistBins.register(): Warning - registered all %d, but still have %d masked values" % (self.numBins, len(np.nonzero(self.Bins.mask))))
         # not reached
     if self.Debug == True:
         self.DebugMessages += "HistBins.register(): Registred all %d expected values, now to check them and take the Bin differences" % self.numBins
     self.HaveData = True

 def insertEmptyCol(self, colName, dt):
     """
 Inserts an empty column into the table. 
 """
     self.cols = np.append(self.cols, colName)
     self.colFormat[colName] = str(self.width) + "." + str(self.prec) + "f"
     self.data[colName] = ma.empty((self.nRows,), dtype=dt)
     self.data[colName][:] = ma.masked

def standardize_fill_value(cube):
    """Work around default `fill_value` when obtaining
    `_CubeSignature` (iris) using `lazy_data()` (biggus).
    Warning use only when you DO KNOW that the slices should
    have the same `fill_value`!!!"""
    if ma.isMaskedArray(cube._my_data):
        fill_value = ma.empty(0, dtype=cube._my_data.dtype).fill_value
        cube._my_data.fill_value = fill_value
    return cube

def test_masked_x():
    data = ma.empty((10,), dtype=[('x', float), ('y', float)])
    data['x'] = np.arange(10.)
    data['y'] = np.arange(10.)
    data.mask = False
    data['x'].mask[2] = True
    x2 = np.arange(0.5, 9.5)
    with pytest.raises(ValueError):
        resample(data, 'x', x2, 'y')

def test_masked_all_valid():
    data = ma.empty((10,), dtype=[('x', float), ('y', float)])
    data['x'] = np.arange(10.)
    data['y'] = np.arange(10.)
    data.mask = False
    x2 = np.arange(0.5, 9.5)
    result = resample(data, 'x', x2, 'y')
    assert np.array_equal(result['x'], x2)
    assert np.array_equal(result['y'], x2)

def empty(coordinates, dtype=float, masked=False):
    """Get an empty ``gridded_array`` of dtype ``dtype``."""
    if masked:
        _data = ma.empty([coordinates[dim].size for dim in list(coordinates.keys())],
                                 dtype=dtype)
    else:
        _data = np.empty([coordinates[dim].size for dim in list(coordinates.keys())],
                                 dtype=dtype)
    return gridded_array(_data, coordinates)

 def add_field_like(self, name, like_array):
     """
     Add a new field to the Datamat with the dtype of the
     like_array and the shape of the like_array except for the first
     dimension which will be instead the field-length of this Datamat.
     """
     new_shape = list(like_array.shape)
     new_shape[0] = len(self)
     new_data = ma.empty(new_shape, like_array.dtype)
     new_data.mask = True
     self.add_field(name, new_data)

def test_masked_one_invalid_nearest():
    data = ma.empty((10,), dtype=[('x', float), ('y', float)])
    data['x'] = np.arange(10.)
    data['y'] = np.ones(10)
    data.mask = False
    data['y'].mask[4] = True
    x2 = np.arange(0.25, 9.25)
    result = resample(data, 'x', x2, 'y', kind='nearest')
    assert np.array_equal(result['x'], x2)
    assert np.all(result['y'][:4] == 1)
    assert result['y'].mask[4]
    assert np.all(result['y'][5:] == 1)

    def bootstrap(self):
        first_slice = self.array[0]
        shape = first_slice.shape
        self.running_total = np.zeros(shape, dtype=self.dtype)

        if self.masked:
            first_slice = first_slice.masked_array()
            self.temp = ma.empty(shape, dtype=self.dtype)
            self.running_total += first_slice.filled(0)
            self._bootstrap_mask(first_slice.mask, shape)
        else:
            first_slice = first_slice.ndarray()
            self.temp = np.empty(shape, dtype=self.dtype)
            self.running_total += first_slice

 def get_thetae_profile(self):
     '''
         Function to calculate the theta-e profile.
         
         Parameters
         ----------
         None
         
         Returns
         -------
         Array of theta-e profile
         '''
     thetae = ma.empty(self.pres.shape[0])
     for i in range(len(self.v)):
         thetae[i] = thermo.ctok( thermo.thetae(self.pres[i], self.tmpc[i], self.dwpc[i]) )
     thetae[thetae == self.missing] = ma.masked
     thetae.set_fill_value(self.missing)
     return thetae

 def get_wetbulb_profile(self):
     '''
         Function to calculate the wetbulb profile.
         
         Parameters
         ----------
         None
         
         Returns
         -------
         Array of wet bulb profile
         '''
     
     wetbulb = ma.empty(self.pres.shape[0])
     for i in range(len(self.v)):
         wetbulb[i] = thermo.wetbulb( self.pres[i], self.tmpc[i], self.dwpc[i] )
     wetbulb[wetbulb == self.missing] = ma.masked
     wetbulb.set_fill_value(self.missing)
     return wetbulb

def test_masked_kind_not_supported():
    data = ma.empty((10,), dtype=[('x', float), ('y', float)])
    data['x'] = np.arange(10.)
    data['y'] = np.ones(10)
    data.mask = False
    data['y'].mask[4] = True
    x2 = np.arange(0.25, 9.25)
    with pytest.raises(ValueError):
        resample(data, 'x', x2, 'y', kind='quadratic')
    with pytest.raises(ValueError):
        resample(data, 'x', x2, 'y', kind='cubic')
    with pytest.raises(ValueError):
        resample(data, 'x', x2, 'y', kind=2)
    with pytest.raises(ValueError):
        resample(data, 'x', x2, 'y', kind=3)
    with pytest.raises(ValueError):
        resample(data, 'x', x2, 'y', kind=4)
    with pytest.raises(ValueError):
        resample(data, 'x', x2, 'y', kind=5)

 def _make_data(self, data, dtype=np.dtype('int32'), fill_value=None,
                mask=None, lazy=False, N=3):
     if isinstance(data, Iterable):
         shape = (len(data), N, N)
         data = np.array(data).reshape(-1, 1, 1)
     else:
         shape = (N, N)
     if mask is not None:
         payload = ma.empty(shape, dtype=dtype, fill_value=fill_value)
         payload.data[:] = data
         if isinstance(mask, bool):
             payload.mask = mask
         else:
             payload[mask] = ma.masked
     else:
         payload = np.empty(shape, dtype=dtype)
         payload[:] = data
     if lazy:
         payload = as_lazy_data(payload)
     return payload