def parsenml(nml, parname, fmt=5):


    val = []

    for line in open(nml):
        if len(line.strip()) > 0:
            if line.strip()[0] != '!':
                tmp = line.split('=')
                vname = tmp[0].split('(')[0].strip()
                if parname.upper() == vname.upper():
                    valstr = tmp[1].split('!')[0].strip()

                    if fmt in char_list:
                        val.append( valstr.replace("'", "") )
                    if fmt == 3 or fmt == 4:
                        for entry in valstr.split(','):
                            val.append(np.int32(entry))
                    if fmt == 7:
                        for entry in valstr.split(','):
                            if entry.upper().strip() in ['F', '.F.', 'FALSE', '.FALSE.']:
                                val.append( np.bool(False))
                            else:
                                val.append(np.bool(True))
                    if fmt == 5:
                        for entry in valstr.split(','):
                            val.append( np.float32(entry))

    if fmt in char_list:
        return val
    else:
        return np.array(val)

 def mask_borders(self, num_pixels=1):
     """
     Mask the border of each ASIC, to a width of `num_pixels`.
     
     Parameters
     ----------
     num_pixels : int
         The size of the border region to mask.
     """
     
     print "Masking %d pixels around the border of each 2x1" % num_pixels
     
     n = int(num_pixels)        
     m = self._blank_mask()
     
     if (num_pixels < 0) or (num_pixels > 194):
         raise ValueError('`num_pixels` must be >0, <194')
     
     for i in range(4):
         for j in range(16):
             
             # mask along the y-dim
             m[i,j,:,0:n] = np.bool(False)
             m[i,j,:,194-n:194] = np.bool(False)
             
             # mask along the x-dim
             m[i,j,0:n,:] = np.bool(False)
             m[i,j,185-n:185,:] = np.bool(False)
             
             # # mask a bar along y in the middle of the 2x1
             # m[i,j,:,194-n:194+n] = np.bool(False)
     
     self._inject_mask('border', m, override_previous=True)
     
     return

 def test_numpy(self):
     """NumPy objects get serialized to readable JSON."""
     l = [
         np.float32(12.5),
         np.float64(2.0),
         np.float16(0.5),
         np.bool(True),
         np.bool(False),
         np.bool_(True),
         np.unicode_("hello"),
         np.byte(12),
         np.short(12),
         np.intc(-13),
         np.int_(0),
         np.longlong(100),
         np.intp(7),
         np.ubyte(12),
         np.ushort(12),
         np.uintc(13),
         np.ulonglong(100),
         np.uintp(7),
         np.int8(1),
         np.int16(3),
         np.int32(4),
         np.int64(5),
         np.uint8(1),
         np.uint16(3),
         np.uint32(4),
         np.uint64(5),
     ]
     l2 = [l, np.array([1, 2, 3])]
     roundtripped = loads(dumps(l2, cls=EliotJSONEncoder))
     self.assertEqual([l, [1, 2, 3]], roundtripped)

def test_sequence_numpy_boolean(seq):
    expected = [np.bool(True), None, np.bool(False), None]
    arr = pa.array(seq(expected))
    assert len(arr) == 4
    assert arr.null_count == 2
    assert arr.type == pa.bool_()
    assert arr.to_pylist() == expected

    def clean_manifold(self):
        current_status = M.heman.get_manifold_status_bits()
        noof_cleans = np.int(self.spinBox_noof_cleans.value())
        print(noof_cleans)
        self.label_cleaning_status.setText("<font style='color: %s'>%s</font>"%('Red', "Cleaning..."))
        gui.QApplication.processEvents()
        M.heman.clean_manifold(noof_cleans)

        # Leave the manifold as we started
        M.heman.set_gas(np.bool(current_status[0]))
        M.heman.set_pump(np.bool(current_status[1]))
        M.heman.set_cryostat(np.bool(current_status[2]))

        self.set_heman_labels()
        self.label_cleaning_status.setText("<font style='color: %s'>%s</font>"%('Black', "Cleaning completed"))

def readData(dataFile):

# read the data sets
# each line has one utterance that contains tab separated utterance words and corresponding IOB tags
# if the input is multiturn session data, the flag following the IOB tags is 1 (session start) or 0 (not session start)
 
	utterances = list()
	tags = list()
	starts = list()
	startid = list()

	# reserving index 0 for padding
	# reserving index 1 for unknown word and tokens
	word_vocab_index = 2
	tag_vocab_index = 2
	word2id = {'<pad>': 0, '<unk>': 1}
	tag2id = {'<pad>': 0, '<unk>': 1}
	id2word = ['<pad>', '<unk>']
	id2tag = ['<pad>', '<unk>']

	utt_count = 0
	temp_startid = 0
	for line in open(dataFile, 'r'):
		d=line.split('\t')
		utt = d[0].strip()
		t = d[1].strip()
		if len(d) > 2:
			start = np.bool(int(d[2].strip()))
			starts.append(start)
			if start:
				temp_startid = utt_count
			startid.append(temp_startid)
		#print 'utt: %s, tags: %s' % (utt,t) 

		temp_utt = list()
		temp_tags = list()
		mywords = utt.split()
		mytags = t.split()
		if len(mywords) != len(mytags):
			print mywords
			print mytags
		# now add the words and tags to word and tag dictionaries
		# also save the word and tag sequence in training data sets
		for i in xrange(len(mywords)):
			if mywords[i] not in word2id:
				word2id[mywords[i]] = word_vocab_index
				id2word.append(mywords[i])
				word_vocab_index += 1
			if mytags[i] not in tag2id:
				tag2id[mytags[i]] = tag_vocab_index
				id2tag.append(mytags[i])
				tag_vocab_index += 1
			temp_utt.append(word2id[mywords[i]])
			temp_tags.append(tag2id[mytags[i]])
		utt_count += 1
		utterances.append(temp_utt)
		tags.append(temp_tags)

	data = {'start': starts, 'startid': startid, 'utterances': utterances, 'tags': tags, 'uttCount': utt_count, 'id2word':id2word, 'id2tag':id2tag, 'wordVocabSize' : word_vocab_index, 'tagVocabSize': tag_vocab_index, 'word2id': word2id, 'tag2id':tag2id}
	return data

 def set_array(self, value):
     """calls set_array and _set_buffer"""
     #if defined as scalar variable, set the array and the buffer to a single numpy value, according to specified type
     if (self._addspc == 'scalar') and (isinstance(value, int) or isinstance(value, long) or isinstance(value, float)):
         if (self._dtypestr == 'int'):
             self._array = np.int32(value)
         elif (self._dtypestr == 'uint'):
             self._array = np.uint32(value)
         elif (self._dtypestr == 'long'):
             self._array = np.int64(value)
         elif (self._dtypestr == 'bool'):
             self._array = np.bool(value)
         elif (self._dtypestr == 'real'):
             if bool(self._solverobj.cldevice.get_info(cl.device_info.DOUBLE_FP_CONFIG)):
                 self._array = np.float64(value)
             else:
                 self._array = np.float32(value)
         self._set_buffer(self._array)
     elif (self._addspc == '__local'):
         raise ValueError(value, '__local defined variables cannot be set from the host device.')
     #if defined as __global, input must match the variable's type, that is: numpy array's dytpe and shape[1], 
     # which defines the specific vector length (real, real4, ...)
     elif isinstance(value, np.ndarray):
         if (len(value.shape) == 1):
             value.shape = (value.shape[0], 1)
         if (value.shape[1] == self._array.shape[1]):
             self._array = value.astype(self._array.dtype, copy=True)
             #self._array = np.zeros(value.shape, value.dtype)
             #self._array += value
             self.set_length(value.shape[0])
             #self._set_buffer(cl.Buffer(self._solverobj.clcontext, cl.mem_flags.READ_WRITE | cl.mem_flags.COPY_HOST_PTR, hostbuf=self._array)) #not needed, called in set_length            
         else:
             raise ValueError(value,'is not a valid value for this variable. Must be a numpy array of fitting shape and dtype or number for scalar varibales. ')
     else:
             raise ValueError(value,'is not a valid value for this variable. Must be a numpy array. ')

def induct(x):
    from . import cudata
    """Compute Copperhead type of an input, also convert data structure"""
    if isinstance(x, cudata.cuarray):
        return (conversions.back_to_front_type(x.type), x)
    if isinstance(x, np.ndarray):
        induced = cudata.cuarray(x)
        return (conversions.back_to_front_type(induced.type), induced)
    if isinstance(x, np.float32):
        return (coretypes.Float, x)
    if isinstance(x, np.float64):
        return (coretypes.Double, x)
    if isinstance(x, np.int32):
        return (coretypes.Int, x)
    if isinstance(x, np.int64):
        return (coretypes.Long, x)
    if isinstance(x, np.bool):
        return (coretypes.Bool, x)
    if isinstance(x, list):
        induced = cudata.cuarray(np.array(x))
        return (conversions.back_to_front_type(induced.type), induced)
    if isinstance(x, float):
        #Treat Python floats as double precision
        return (coretypes.Double, np.float64(x))
    if isinstance(x, int):
        #Treat Python ints as 64-bit ints (following numpy)
        return (coretypes.Long, np.int64(x))
    if isinstance(x, bool):
        return (coretypes.Bool, np.bool(x))
    if isinstance(x, tuple):
        sub_types, sub_elements = zip(*(induct(y) for y in x))
        return (coretypes.Tuple(*sub_types), tuple(sub_elements))
    #Can't digest this input
    raise ValueError("This input is not convertible to a Copperhead data structure: %r" % x)

    def _slice(self, bin_centers, bin_values):
        """
        slice out only the requested parts of the radial profile
        """

        rmin = np.min( self.radius_range )
        rmax = np.max( self.radius_range )
        if (not np.any(bin_centers > rmax)) or (not np.any(bin_centers < rmin)):
            raise ValueError('Invalid radius range -- out of bounds of measured radii.')
        
        if len(self.radius_range) > 0:
            include = np.zeros( len(bin_values), dtype=np.bool)
            for i in range(0, len(self.radius_range), 2):
                inds = (bin_centers > self.radius_range[i]) * \
                       (bin_centers < self.radius_range[i+1])
                include[inds] = np.bool(True)

            if np.sum(include) == 0:
                raise RuntimeError('`radius_range` slices were not big enough to '
                                   'include any data!')

            bin_centers = bin_centers[include]
            bin_values  = bin_values[include]
            
        return bin_centers, bin_values

def test_deltaSoft(n=1):

    p = 1000
    ctype = np.zeros(4)
    for i in range(n):
        v = np.random.normal(0, 10, p)
        normalize = np.bool(np.sign(np.random.normal()) + 1)
        if normalize:
            v = v / np.sqrt(np.sum(v ** 2))
        l = np.max([np.fabs(np.random.normal(np.sum(np.fabs(v)), 50)), 1.0])
        vec = scca.deltaSoft(v, l, normalize)
        if normalize:
            n2vec = np.sum(vec ** 2)
            assert np.fabs(n2vec - 1.0) < 1e-8
        n1v = np.sum(np.fabs(v))
        if n1v <= l:
            if normalize:
                ctype[0] += 1
            else:
                ctype[1] += 1
            assert np.sum((v - vec) ** 2) < 1e-8
        else:
            if normalize:
                ctype[2] += 1
            else:
                ctype[3] += 1
            n1vec = np.sum(np.fabs(vec))
            assert np.sum((n1vec - l) ** 2) < 1e-8

def printout(str, it_num = False):
    '''
    Prints iteration progress number or other information in one line of 
    terminal.

    Parameters
    ----------
    str:    string
            String defining what will be printed.
    it_num: integer 
            Iteration number to be printed. If False, will print out 
            contents of str instead.

    Returns
    -------
    None
    '''

    # Create print-out for terminal that can be overwritten
    stdout.write('\r\n')
    if np.bool(it_num):
        # Print iteration number
        stdout.write(str % it_num)
    else:
        # Print other input
        stdout.write(str)
    
    # Clear printed value to allow overwriting for next
    stdout.flush()

    def test_is_number(self):

        self.assertTrue(is_number(True))
        self.assertTrue(is_number(1))
        self.assertTrue(is_number(1.1))
        self.assertTrue(is_number(1 + 3j))
        self.assertTrue(is_number(np.bool(False)))
        self.assertTrue(is_number(np.int64(1)))
        self.assertTrue(is_number(np.float64(1.1)))
        self.assertTrue(is_number(np.complex128(1 + 3j)))
        self.assertTrue(is_number(np.nan))

        self.assertFalse(is_number(None))
        self.assertFalse(is_number('x'))
        self.assertFalse(is_number(datetime(2011, 1, 1)))
        self.assertFalse(is_number(np.datetime64('2011-01-01')))
        self.assertFalse(is_number(Timestamp('2011-01-01')))
        self.assertFalse(is_number(Timestamp('2011-01-01',
                                             tz='US/Eastern')))
        self.assertFalse(is_number(timedelta(1000)))
        self.assertFalse(is_number(Timedelta('1 days')))

        # questionable
        self.assertFalse(is_number(np.bool_(False)))
        self.assertTrue(is_number(np.timedelta64(1, 'D')))

 def parse_value(self,value):
     '''parses and casts the raw value into an acceptable format for __value
     lot of defense here, so we can make assumptions later
     '''
     if isinstance(value,list):
         print 'util_2d: casting list to array'
         value = np.array(value)
     if isinstance(value,bool):
         if self.dtype == np.bool:
             try:
                 value = np.bool(value)
                 return value
             except:
                 raise Exception('util_2d:could not cast '+\
                     'boolean value to type "np.bool": '+str(value))
         else:
             raise Exeception('util_2d:value type is bool, '+\
                ' but dtype not set as np.bool') 
     if isinstance(value,str):
         if self.dtype == np.int:
             try:
                 value = int(value)
             except:
                 assert os.path.exists(value),'could not find file: '+str(value)
                 return value
         else:
             try:
                 value = float(value)
             except:
                 assert os.path.exists(value),'could not find file: '+str(value)
                 return value
     if np.isscalar(value):
         if self.dtype == np.int:
             try:
                 value = np.int(value)
                 return value
             except:
                 raise Exception('util_2d:could not cast scalar '+\
                     'value to type "int": '+str(value))
         elif self.dtype == np.float32:
             try:
                 value = np.float32(value)
                 return value
             except:
                 raise Exception('util_2d:could not cast '+\
                     'scalar value to type "float": '+str(value))
         
     if isinstance(value,np.ndarray):
         if self.shape != value.shape:
             raise Exception('util_2d:self.shape: '+str(self.shape)+\
                 ' does not match value.shape: '+str(value.shape))
         if self.dtype != value.dtype:
             print 'util_2d:warning - casting array of type: '+\
                 str(value.dtype)+' to type: '+str(self.dtype)
         return value.astype(self.dtype)
     
     else:
         raise Exception('util_2d:unsupported type in util_array: '\
             +str(type(value))) 

    def _pilatus_mask(self, border_size=3):
        """
        The pixels on the edges of the detector are often noisy -- this function
        provides a way to mask both the gaps and these border pixels.

        Parameters
        ----------
        border_size : int
            The size of the border (in pixels) with which to extend the mask
            around the detector gaps.
        """

        border_size = int(border_size)
        mask = np.ones(self.intensities_shape, dtype=np.bool)

        # below we have the cols (x_gaps) and rows (y_gaps) to mask
        # these mask the ASIC gaps

        x_gaps = [(194-border_size,  212+border_size),
                  (406-border_size,  424+border_size),
                  (618-border_size,  636+border_size),
                  (830-border_size,  848+border_size),
                  (1042-border_size, 1060+border_size),
                  (1254-border_size, 1272+border_size),
                  (1466-border_size, 1484+border_size),
                  (1678-border_size, 1696+border_size),
                  (1890-border_size, 1908+border_size),
                  (2102-border_size, 2120+border_size),
                  (2314-border_size, 2332+border_size)]
                  
        y_gaps = [(486-border_size,  494+border_size),
                  (980-border_size,  988+border_size),
                  (1474-border_size, 1482+border_size),
                  (1968-border_size, 1976+border_size)]

        for x in x_gaps:
            mask[x[0]:x[1],:] = np.bool(False)

        for y in y_gaps:
            mask[:,y[0]:y[1]] = np.bool(False)
        
            
        # we also mask the beam stop for 12-2...
        mask[1200:1325,1164:] = np.bool(False)

        return mask

 def test_int(self):
     self.assert_equal_with_lambda_check(_flexible_type(1), 1)
     self.assert_equal_with_lambda_check(_flexible_type(1L), 1)
     self.assert_equal_with_lambda_check(_flexible_type(True), 1)
     self.assert_equal_with_lambda_check(_flexible_type(False), 0)
     # numpy types
     self.assert_equal_with_lambda_check(_flexible_type(np.int_(1)), 1)
     self.assert_equal_with_lambda_check(_flexible_type(np.int64(1)), 1)
     self.assert_equal_with_lambda_check(_flexible_type(np.int32(1)), 1)
     self.assert_equal_with_lambda_check(_flexible_type(np.int16(1)), 1)
     self.assert_equal_with_lambda_check(_flexible_type(np.uint64(1)), 1)
     self.assert_equal_with_lambda_check(_flexible_type(np.uint32(1)), 1)
     self.assert_equal_with_lambda_check(_flexible_type(np.uint16(1)), 1)
     self.assert_equal_with_lambda_check(_flexible_type(np.bool(1)), 1)
     self.assert_equal_with_lambda_check(_flexible_type(np.bool(0)), 0)
     self.assert_equal_with_lambda_check(_flexible_type(np.bool_(1)), 1)
     self.assert_equal_with_lambda_check(_flexible_type(np.bool_(0)), 0)
     self.assert_equal_with_lambda_check(_flexible_type(np.bool8(1)), 1)
     self.assert_equal_with_lambda_check(_flexible_type(np.bool8(0)), 0)

def printout(str, it_num = False):
    '''
    Used to print iteration progress number.
    '''
    stdout.write('\r\n')
    if np.bool(it_num):
        stdout.write(str % it_num)
    else:
        stdout.write(str)
    stdout.flush()

    def _convert_metadata_vector(self, values):
        """Convert a list of numpy data into a numpy array with the better
        fitting type."""
        converted = []
        types = set([])
        has_none = False
        for v in values:
            if v is None:
                converted.append(None)
                has_none = True
            else:
                c = self._convert_value(v)
                converted.append(c)
                types.add(c.dtype)

        if has_none and len(types) == 0:
            # That's a list of none values
            return numpy.array([0] * len(values), numpy.int8)

        result_type = numpy.result_type(*types)

        if issubclass(result_type.type, numpy.string_):
            # use the raw data to create the array
            result = values
        elif issubclass(result_type.type, numpy.unicode_):
            # use the raw data to create the array
            result = values
        else:
            result = converted

        result_type = self._normalize_vector_type(result_type)

        if has_none:
            # Fix missing data according to the array type
            if result_type.kind == "S":
                none_value = b""
            elif result_type.kind == "U":
                none_value = u""
            elif result_type.kind == "f":
                none_value = numpy.float("NaN")
            elif result_type.kind == "i":
                none_value = numpy.int(0)
            elif result_type.kind == "u":
                none_value = numpy.int(0)
            elif result_type.kind == "b":
                none_value = numpy.bool(False)
            else:
                none_value = None

            for index, r in enumerate(result):
                if r is not None:
                    continue
                result[index] = none_value

        return numpy.array(result, dtype=result_type)

def processUDPMessage(m,rc):

    l = re.split(',', m)
    print l

    if not len(l) == 8:
        return False

    rc.header.frame_id = "agent_" + str(g_agent_number)
    rc.header.stamp = rospy.Time.now()
    rc.velocity.linear.x = float(l[0])
    rc.velocity.linear.y = float(l[1])
    rc.velocity.angular.z = float(l[2])
    rc.kick_power = np.uint8(l[3])
    rc.high_kick = np.bool(np.uint8(l[4]))
    rc.low_kick = np.bool(np.uint8(l[5]))
    rc.grabber_left_speed = np.uint8(l[6])
    rc.grabber_right_speed = np.uint8(l[7])

    return True

    def test_polar_mask_conversion(self):

        # make a real mask that is a circle, and then it should be easy to check
        # that the polar mask is correct
        
        q_cutoff_index = 5
        q_values = np.arange(3.5, 4.5, 0.1)
        q_cutoff = q_values[q_cutoff_index]
        
        rm = np.ones(self.d.num_pixels, dtype=np.bool)
        rm[self.d.recpolar[:,0] < q_cutoff] = np.bool(False)
        
        shot = xray.Shotset(self.i, self.d, mask=rm)
        r = shot.to_rings(q_values)
        
        ref_pm = np.ones((len(q_values), r.num_phi), dtype=np.bool)
        ref_pm[:q_cutoff_index+1,:] = np.bool(False)
        
        print "num px masked", ref_pm.sum(), r.polar_mask.sum()
        assert ref_pm.sum() == r.polar_mask.sum() # same num px masked
        assert_array_equal(ref_pm, r.polar_mask)

def gethost_our_morphology_data():
    """
    This is a compilation for some SNe~Ia (nearby once)
      It is mainly based on Asiago, but Greg Aldering made some
    """
    dico = {}
    datas = [l for l in open(_fileMorphoCompil).read().splitlines() if l[0] != "#"]
    for l in datas:
        A = l.split()
        dico_ = {
            "object":A[0],
            "zcmb":np.float(A[1]),
            "inPS1": np.bool(np.int(A[2])),
            "inUn3": np.bool(np.int(A[3])),
            "inK14": np.bool(np.int(A[4])),
            "inR14": np.bool(np.int(A[5])),
            "Fchart":np.bool(np.int(A[6])),
            "Ra":np.float(A[7].split(",")[0]),"Dec":np.float(A[8].split(",")[0]),
            "morphology":A[9]
            }
        dico[dico_["object"]] = dico_
    return dico