def test_init(self):
        import numpy as np
        import math
        import sys

        assert np.intp() == np.intp(0)
        assert np.intp("123") == np.intp(123)
        raises(TypeError, np.intp, None)
        assert np.float64() == np.float64(0)
        assert math.isnan(np.float64(None))
        assert np.bool_() == np.bool_(False)
        assert np.bool_("abc") == np.bool_(True)
        assert np.bool_(None) == np.bool_(False)
        assert np.complex_() == np.complex_(0)
        # raises(TypeError, np.complex_, '1+2j')
        assert math.isnan(np.complex_(None))
        for c in ["i", "I", "l", "L", "q", "Q"]:
            assert np.dtype(c).type().dtype.char == c
        for c in ["l", "q"]:
            assert np.dtype(c).type(sys.maxint) == sys.maxint
        for c in ["L", "Q"]:
            assert np.dtype(c).type(sys.maxint + 42) == sys.maxint + 42
        assert np.float32(np.array([True, False])).dtype == np.float32
        assert type(np.float32(np.array([True]))) is np.ndarray
        assert type(np.float32(1.0)) is np.float32
        a = np.array([True, False])
        assert np.bool_(a) is a

def test_working_type():
    # Which type do input types with slope and inter cast to in numpy?
    # Wrapper function because we need to use the dtype str for comparison.  We
    # need this because of the very confusing np.int32 != np.intp (on 32 bit).
    def wt(*args, **kwargs):
        return np.dtype(working_type(*args, **kwargs)).str
    d1 = np.atleast_1d
    for in_type in NUMERIC_TYPES:
        in_ts = np.dtype(in_type).str
        assert_equal(wt(in_type), in_ts)
        assert_equal(wt(in_type, 1, 0), in_ts)
        assert_equal(wt(in_type, 1.0, 0.0), in_ts)
        in_val = d1(in_type(0))
        for slope_type in NUMERIC_TYPES:
            sl_val = slope_type(1) # no scaling, regardless of type
            assert_equal(wt(in_type, sl_val, 0.0), in_ts)
            sl_val = slope_type(2) # actual scaling
            out_val = in_val / d1(sl_val)
            assert_equal(wt(in_type, sl_val), out_val.dtype.str)
            for inter_type in NUMERIC_TYPES:
                i_val = inter_type(0) # no scaling, regardless of type
                assert_equal(wt(in_type, 1, i_val), in_ts)
                i_val = inter_type(1) # actual scaling
                out_val = in_val - d1(i_val)
                assert_equal(wt(in_type, 1, i_val), out_val.dtype.str)
                # Combine scaling and intercept
                out_val = (in_val - d1(i_val)) / d1(sl_val)
                assert_equal(wt(in_type, sl_val, i_val), out_val.dtype.str)
    # Confirm that type codes and dtypes work as well
    f32s = np.dtype(np.float32).str
    assert_equal(wt('f4', 1, 0), f32s)
    assert_equal(wt(np.dtype('f4'), 1, 0), f32s)

def test_dtype(seed=1234):
    np.random.seed(seed)

    dtype = [
        ("coords", np.float64, (4, )),
        ("log_prior", np.float64),
        ("log_likelihood", np.float64),
        ("accepted", bool)
    ]

    coords = np.random.randn(4)
    state = State(coords)
    assert state.dtype == np.dtype(dtype)

    state = State(coords, face=10.0, blah=6, _hidden=None)
    dtype += [
        ("blah", int),
        ("face", float),
    ]
    assert state.dtype == np.dtype(dtype)

    state = State(coords, face=10.0, blah=6, _hidden=None,
                  matrix=np.ones((3, 1)))
    dtype += [
        ("matrix", float, (3, 1)),
    ]
    assert state.dtype == np.dtype(dtype)

    state = State(coords, face=10.0, blah=6, _hidden=None,
                  matrix=np.ones((3, 1)), vector=np.zeros(3))
    dtype += [
        ("vector", float, (3,)),
    ]
    assert state.dtype == np.dtype(dtype)

def test_issue321():
    """L-PICOLA outputs single-precision with no mass block, which causes problems
    with testing kd-trees"""

    f = pynbody.load("testdata/lpicola/lpicola_z0p000.0")
    assert f['pos'].dtype==np.dtype('float32')
    assert f['mass'].dtype==np.dtype('float32')

    def dtype(self):
        # Image data types (Image Object chapter on DM help)#
        # key = DM data type code
        # value = numpy data type
        if self.imdict.ImageData.DataType == 4:
            raise NotImplementedError(
                "Reading data of this type is not implemented.")

        imdtype_dict = {
            0: 'not_implemented',  # null
            1: 'int16',
            2: 'float32',
            3: 'complex64',
            5: 'float32',  # not numpy: 8-Byte packed complex (FFT data)
            6: 'uint8',
            7: 'int32',
            8: np.dtype({'names': ['B', 'G', 'R', 'A'],
                         'formats': ['u1', 'u1', 'u1', 'u1']}),
            9: 'int8',
            10: 'uint16',
            11: 'uint32',
            12: 'float64',
            13: 'complex128',
            14: 'bool',
            23: np.dtype({'names': ['B', 'G', 'R', 'A'],
                          'formats': ['u1', 'u1', 'u1', 'u1']}),
            27: 'complex64',  # not numpy: 8-Byte packed complex (FFT data)
            28: 'complex128',  # not numpy: 16-Byte packed complex (FFT data)
        }
        return imdtype_dict[self.imdict.ImageData.DataType]

def test_rasterize_supported_dtype(basic_geometry):
    """ Supported data types should return valid results """

    with Env():
        supported_types = (
            ('int16', -32768),
            ('int32', -2147483648),
            ('uint8', 255),
            ('uint16', 65535),
            ('uint32', 4294967295),
            ('float32', 1.434532),
            ('float64', -98332.133422114)
        )

        for dtype, default_value in supported_types:
            truth = np.zeros(DEFAULT_SHAPE, dtype=dtype)
            truth[2:4, 2:4] = default_value

            result = rasterize(
                [basic_geometry],
                out_shape=DEFAULT_SHAPE,
                default_value=default_value,
                dtype=dtype
            )
            assert np.array_equal(result, truth)
            assert np.dtype(result.dtype) == np.dtype(truth.dtype)

            result = rasterize(
                [(basic_geometry, default_value)],
                out_shape=DEFAULT_SHAPE
            )
            if np.dtype(dtype).kind == 'f':
                assert np.allclose(result, truth)
            else:
                assert np.array_equal(result, truth)

def CfxCentreLineSnapshot(filename):
    """Factory function wrapping a CFX snapshot.
    
    Load the data with:
    >>> snap = CfxSnapshot(filename)

    Fields are constructed from the header line.
    
        
    """
    (__raw_row, fieldUnits) = parseHeader(filename, AllData=True)
    __raw_row = [('id', int),] + __raw_row
    fieldUnits['id'] = 1
    
                 # ('position', float, (3,)),
                 # ('strain_rate', float),
                 # ('speed', float),
                 # ('velocity', float, (3,)),
                 # ('wall_shear', float, (4,))]

    __readable_row = np.dtype(__raw_row[1:])
    row = np.dtype(__raw_row)
    
    noindex = np.genfromtxt(filename, skip_header=findStart(filename, AllData=True)+2,
                           delimiter=',',
                           dtype=__readable_row).view(np.recarray)
    index = np.recarray(shape=noindex.shape, dtype=row)
    index.id = np.arange(len(noindex))
    for el in __raw_row[1:]:
        key = el[0]
        index.__setattr__(key, U.convert(noindex.__getattribute__(key), fieldUnits[key], hlbUnits[key]))
        continue
    
    return index

def main(fname):

    basename = os.path.basename(fname)[0:-4]
    abspath = os.path.dirname(os.path.abspath(fname))
    output_fname = os.path.join(abspath, basename+'.hdf5')

    if 'mdr1_fofp_' in basename:
        print("\n...Reading particle data...\n")
        dt = np.dtype([('rowid', 'i8'), 
    ('x', 'f8'), ('y', 'f8'), ('z','f8'),
    ('vx', 'f8'), ('vy', 'f8'), ('vz','f8'), ('haloid', 'i8')]
    )
    else:
        print("\n...Reading halo data...\n")
        dt = np.dtype([('rowid', 'i8'), ('haloid', 'i8'), 
    ('x', 'f8'), ('y', 'f8'), ('z','f8'),
    ('vx', 'f8'), ('vy', 'f8'), ('vz','f8'), ('mass', 'f4'), ('size', 'f4')])

    start = time()
    data = read_csv(fname, dt)
    end = time()
    runtime = end-start
    print("Total time to read csv = %.1f seconds\n" % runtime)

    with h5py.File(output_fname,'w') as f:
        f['data'] = data

 def test_masked_all(self):
     # Tests masked_all
     # Standard dtype
     test = masked_all((2,), dtype=float)
     control = array([1, 1], mask=[1, 1], dtype=float)
     assert_equal(test, control)
     # Flexible dtype
     dt = np.dtype({'names': ['a', 'b'], 'formats': ['f', 'f']})
     test = masked_all((2,), dtype=dt)
     control = array([(0, 0), (0, 0)], mask=[(1, 1), (1, 1)], dtype=dt)
     assert_equal(test, control)
     test = masked_all((2, 2), dtype=dt)
     control = array([[(0, 0), (0, 0)], [(0, 0), (0, 0)]],
                     mask=[[(1, 1), (1, 1)], [(1, 1), (1, 1)]],
                     dtype=dt)
     assert_equal(test, control)
     # Nested dtype
     dt = np.dtype([('a', 'f'), ('b', [('ba', 'f'), ('bb', 'f')])])
     test = masked_all((2,), dtype=dt)
     control = array([(1, (1, 1)), (1, (1, 1))],
                     mask=[(1, (1, 1)), (1, (1, 1))], dtype=dt)
     assert_equal(test, control)
     test = masked_all((2,), dtype=dt)
     control = array([(1, (1, 1)), (1, (1, 1))],
                     mask=[(1, (1, 1)), (1, (1, 1))], dtype=dt)
     assert_equal(test, control)
     test = masked_all((1, 1), dtype=dt)
     control = array([[(1, (1, 1))]], mask=[[(1, (1, 1))]], dtype=dt)
     assert_equal(test, control)

    def test_pass_dtype(self):
        data = """\
one,two
1,a
2,b
3,c
4,d"""

        def _make_reader(**kwds):
            return TextReader(StringIO(data), delimiter=',', **kwds)

        reader = _make_reader(dtype={'one': 'u1', 1: 'S1'})
        result = reader.read()
        self.assertEqual(result[0].dtype, 'u1')
        self.assertEqual(result[1].dtype, 'S1')

        reader = _make_reader(dtype={'one': np.uint8, 1: object})
        result = reader.read()
        self.assertEqual(result[0].dtype, 'u1')
        self.assertEqual(result[1].dtype, 'O')

        reader = _make_reader(dtype={'one': np.dtype('u1'),
                                     1: np.dtype('O')})
        result = reader.read()
        self.assertEqual(result[0].dtype, 'u1')
        self.assertEqual(result[1].dtype, 'O')

  def testMakeTableExceptions(self):
    # Verify that contents is being type-checked and shape-checked.
    with self.assertRaises(ValueError):
      text_plugin.make_table([])

    with self.assertRaises(ValueError):
      text_plugin.make_table('foo')

    with self.assertRaises(ValueError):
      invalid_shape = np.full((3, 3, 3), 'nope', dtype=np.dtype('S3'))
      text_plugin.make_table(invalid_shape)

    # Test headers exceptions in 2d array case.
    test_array = np.full((3, 3), 'foo', dtype=np.dtype('S3'))
    with self.assertRaises(ValueError):
      # Headers is wrong type.
      text_plugin.make_table(test_array, headers='foo')
    with self.assertRaises(ValueError):
      # Too many headers.
      text_plugin.make_table(test_array, headers=['foo', 'bar', 'zod', 'zoink'])
    with self.assertRaises(ValueError):
      # headers is 2d
      text_plugin.make_table(test_array, headers=test_array)

    # Also make sure the column counting logic works in the 1d array case.
    test_array = np.array(['foo', 'bar', 'zod'])
    with self.assertRaises(ValueError):
      # Too many headers.
      text_plugin.make_table(test_array, headers=test_array)

    def __mul__(self, b):
        if type(b) is not np.ndarray:
            raise TypeError('Can only multiply by a numpy array.')

        if len(b.shape) == 1 or b.shape[1] == 1:
            b = b.flatten()
            # Just one RHS

            if b.dtype is np.dtype('O'):
                b = b.astype(type(b[0]))

            if factorize:
                X = self.solver.solve(b, **self.kwargs)
            else:
                X = fun(self.A, b, **self.kwargs)
        else: # Multiple RHSs
            if b.dtype is np.dtype('O'):
                b = b.astype(type(b[0,0]))

            X = np.empty_like(b)

            for i in range(b.shape[1]):
                if factorize:
                    X[:,i] = self.solver.solve(b[:,i])
                else:
                    X[:,i] = fun(self.A, b[:,i], **self.kwargs)

        if self.checkAccuracy:
            _checkAccuracy(self.A, b, X, self.accuracyTol)
        return X

    def get_default_dtype(structured=True):
        if structured:
            dtype = np.dtype([("k", np.int), ("i", np.int), ("j", np.int),
                              ("segment", np.int), ("reach", np.int),
                              ("flow", np.float32), ("stage", np.float32),
                              ("cond", np.float32), ("sbot", np.float32),
                              ("stop", np.float32),
                              ("width", np.float32), ("slope", np.float32),
                              ("rough", np.float32)])
        else:
            dtype = np.dtype([("node", np.int),
                              ("segment", np.int), ("reach", np.int),
                              ("flow", np.float32), ("stage", np.float32),
                              ("cond", np.float32), ("sbot", np.float32),
                              ("stop", np.float32),
                              ("width", np.float32), ("slope", np.float32),
                              ("rough", np.float32)])

        dtype2 = np.dtype([("itrib01", np.int), ("itrib02", np.int),
                           ("itrib03", np.int), ("itrib04", np.int),
                           ("itrib05", np.int), ("itrib06", np.int),
                           ("itrib07", np.int), ("itrib08", np.int),
                           ("itrib09", np.int), ("itrib10", np.int),
                           ("iupseg", np.int)])
        return dtype, dtype2

    def test_fromMultipleArrays(self):
        ary = arange(8, dtype=dtype('int16')).reshape((2, 4))
        ary2 = arange(8, 16, dtype=dtype('int16')).reshape((2, 4))

        series = SeriesLoader(self.sc).fromArrays([ary, ary2])

        seriesvals = series.collect()
        seriesary = series.pack()

        # check ordering of keys
        assert_equals((0, 0), seriesvals[0][0])  # first key
        assert_equals((1, 0), seriesvals[1][0])  # second key
        assert_equals((3, 0), seriesvals[3][0])
        assert_equals((0, 1), seriesvals[4][0])
        assert_equals((3, 1), seriesvals[7][0])

        # check dimensions tuple is reversed from numpy shape
        assert_equals(ary.shape[::-1], series.dims.count)

        # check that values are in original order, with subsequent point concatenated in values
        collectedvals = array([kv[1] for kv in seriesvals], dtype=dtype('int16'))
        assert_true(array_equal(ary.ravel(), collectedvals[:, 0]))
        assert_true(array_equal(ary2.ravel(), collectedvals[:, 1]))

        # check that packing returns concatenation of input arrays, with time as first dimension
        assert_true(array_equal(ary.T, seriesary[0]))
        assert_true(array_equal(ary2.T, seriesary[1]))

    def test_fromArrays(self):
        ary = arange(8, dtype=dtype('int16')).reshape((2, 4))

        series = SeriesLoader(self.sc).fromArrays(ary)

        seriesvals = series.collect()
        seriesary = series.pack()

        # check ordering of keys
        assert_equals((0, 0), seriesvals[0][0])  # first key
        assert_equals((1, 0), seriesvals[1][0])  # second key
        assert_equals((2, 0), seriesvals[2][0])
        assert_equals((3, 0), seriesvals[3][0])
        assert_equals((0, 1), seriesvals[4][0])
        assert_equals((1, 1), seriesvals[5][0])
        assert_equals((2, 1), seriesvals[6][0])
        assert_equals((3, 1), seriesvals[7][0])

        # check dimensions tuple is reversed from numpy shape
        assert_equals(ary.shape[::-1], series.dims.count)

        # check that values are in original order
        collectedvals = array([kv[1] for kv in seriesvals], dtype=dtype('int16')).ravel()
        assert_true(array_equal(ary.ravel(), collectedvals))

        # check that packing returns transpose of original array
        assert_true(array_equal(ary.T, seriesary))

    def ReadFilePart(self):
        ## Setup for reading file by phase, to (hopefully) save
        # memory. 

        ## Save a file handle somewheer
        self._file_handle = open(self._filename, "rb")
        header = self._file_handle.read(16)
        count, headerSize = struct.unpack_from('QQ',header,0)
        print "Count: " + str(count) + ",Header Size:" + str(headerSize)
        dt = np.dtype([("TotalTime",float,1),
                     ("MallocTime",float,1),
                     ("MallocSize",int,1),
                     ("GPUTime",float,1),
                     ("GPUAvg", float, 1),
                     ("MemcpyTime",float,1),
                     ("Memcpysize",int,1),
                     ("TotalMem",int,1),
                     ("TotalMemRead",int,1),
                     ("TotalMemWrite",int,1)])
        self._kdt = np.dtype([("TotalTime",float,1),
                     ("TBytesRead",int,1),
                     ("TBytesWritten",int,1),
                     ("CBytesRead",int,1),
                     ("CBytesWritten", int, 1)])
        self._phases = np.zeros((count), dtype=dt)
        self._kernels = []
        self._phase_offsets = []
        self._residentPhase = -1
        self.ReadHeader(count,headerSize)

def get_trace(f, num_points, big):
    """
    Get a trace from an open RNMRTK file.

    Parameters
    -----------
    f : file object
        Open file object to read from.
    num_points : int
        Number of points in trace (R+I)
    big : bool
        True for data that is big-endian, False for little-endian.

    Returns
    -------
    trace : ndarray
        Raw trace of NMR data.

    """
    if big:
        bsize = num_points * np.dtype('>f4').itemsize
        return np.frombuffer(f.read(bsize), dtype='>f4')
    else:
        bsize = num_points * np.dtype('<f4').itemsize
        return np.frombuffer(f.read(bsize), dtype='<f4')

    def test_view(self):
        import numpy as np
        import sys

        s = np.dtype("int64").type(12)
        exc = raises(ValueError, s.view, "int8")
        assert exc.value[0] == "new type not compatible with array."
        t = s.view("double")
        assert type(t) is np.double
        assert t < 7e-323
        t = s.view("complex64")
        assert type(t) is np.complex64
        assert 0 < t.real < 1
        assert t.imag == 0
        exc = raises(TypeError, s.view, "string")
        assert exc.value[0] == "data-type must not be 0-sized"
        t = s.view("S8")
        assert type(t) is np.string_
        assert t == "\x0c"
        s = np.dtype("string").type("abc1")
        assert s.view("S4") == "abc1"
        if "__pypy__" in sys.builtin_module_names:
            raises(NotImplementedError, s.view, [("a", "i2"), ("b", "i2")])
        else:
            b = s.view([("a", "i2"), ("b", "i2")])
            assert b.shape == ()
            assert b[0] == 25185
            assert b[1] == 12643
        if "__pypy__" in sys.builtin_module_names:
            raises(TypeError, "np.dtype([('a', 'int64'), ('b', 'int64')]).type('a' * 16)")
        else:
            s = np.dtype([("a", "int64"), ("b", "int64")]).type("a" * 16)
            assert s.view("S16") == "a" * 16

    def test_frame_add_datetime64_col_other_units(self):
        n = 100

        units = ['h', 'm', 's', 'ms', 'D', 'M', 'Y']

        ns_dtype = np.dtype('M8[ns]')

        for unit in units:
            dtype = np.dtype('M8[%s]' % unit)
            vals = np.arange(n, dtype=np.int64).view(dtype)

            df = DataFrame({'ints': np.arange(n)}, index=np.arange(n))
            df[unit] = vals

            ex_vals = to_datetime(vals.astype('O')).values

            self.assertEqual(df[unit].dtype, ns_dtype)
            self.assertTrue((df[unit].values == ex_vals).all())

        # Test insertion into existing datetime64 column
        df = DataFrame({'ints': np.arange(n)}, index=np.arange(n))
        df['dates'] = np.arange(n, dtype=np.int64).view(ns_dtype)

        for unit in units:
            dtype = np.dtype('M8[%s]' % unit)
            vals = np.arange(n, dtype=np.int64).view(dtype)

            tmp = df.copy()

            tmp['dates'] = vals
            ex_vals = to_datetime(vals.astype('O')).values

            self.assertTrue((tmp['dates'].values == ex_vals).all())

 def test_ldexp_overflow(self):
     # silence warning emitted on overflow
     with np.errstate(over="ignore"):
         imax = np.iinfo(np.dtype('l')).max
         imin = np.iinfo(np.dtype('l')).min
         assert_equal(ncu.ldexp(2., imax), np.inf)
         assert_equal(ncu.ldexp(2., imin), 0)