def test_name(self):
     for n in [16, 1, 5, 8, 16, 20, 20 * 8]:
         w = np.random.random(n)
         j = distpy.JaccardWeighted(w)
         j2 = pickle.loads(pickle.dumps(j, -1))
         for x in range(n):
             a = np.zeros(n, dtype=np.uint8)
             b = np.zeros(n, dtype=np.uint8)
             a[x] = 1
             b[x] = 1
             a = np.packbits(a)
             b = np.packbits(b)
             out0 = j.dist(a, b)
             out1 = jaccard_weighted_slow(a, b, w)
             out2 = j2.dist(a, b)
             self.assertEqual(out0, out1)
             self.assertEqual(out0, w[x])
             self.assertEqual(out0, out2)
             print((out0, out1, w[x]))
         for x in range(1000):
             bytes = np.ceil(n / 8.0)
             a = np.fromstring(np.random.bytes(bytes), dtype=np.uint8)
             b = np.fromstring(np.random.bytes(bytes), dtype=np.uint8)
             out0 = j.dist(a, b)
             out1 = jaccard_weighted_slow(a, b, w)
             out2 = j2.dist(a, b)
             self.assertAlmostEqual(out0, out1)
             self.assertEqual(out0, out2)
             print((out0, out1))

def test_packbits_very_large():
    # test some with a larger arrays gh-8637
    # code is covered earlier but larger array makes crash on bug more likely
    for s in range(950, 1050):
        for dt in '?bBhHiIlLqQ':
            x = np.ones((200, s), dtype=bool)
            np.packbits(x, axis=1)

def comprimir(arvore: HuffmanNode, bitGroupSize: int, compressedBitSize, codeTable : dict, dataBuffer):
    global paddingSize, nomeFicheiro
    try:
        outputHandler = open(nomeFicheiro.replace('.pbm', '.cpbm'), 'wb')
        #codificar a arvore de huffman e escreve-la no ficheiro
        writeTreeToFile(outputHandler, compressedBitSize, paddingSize, codeTable, bitGroupSize, arvore)
        
        print("A comprimir os dados com grupos de:", bitGroupSize)
        reiniciarBufferDoFicheiro()
        outBuffer = list()
        get = codeTable.get
        bitpos = 0
        tamanhoFicheiroBits = tamanhoFicheiro * 8
        while(bitpos < tamanhoFicheiroBits):
            #ler um grupo de bits
            bitGroup = read(dataBuffer, bitGroupSize)
            codigo = get(bitGroup, None)
            if(codigo != None):
                outBuffer.extend(codigo)
            bitpos +=bitGroupSize

        print("Padding adicional no fim:", paddingSize)
        np.packbits(outBuffer, -1).tofile(outputHandler)
        outputHandler.close()
    except:
        print("Compressao de ficheiro pbm concluida sem exito")

def elucidate_cc_split(parent_id, split_id):
	parent_id_bytes = numpy.array(tuple(parent_id)).view(dtype = numpy.uint8)
	split_id_bytes = numpy.array(tuple(split_id)).view(dtype = numpy.uint8)

	parent_id_bits = numpy.unpackbits(parent_id_bytes)
	split_id_bits = numpy.unpackbits(split_id_bytes)

	n_parent_bits = len(parent_id_bits)
	n_split_bits = len(split_id_bits)

	child1_bits = numpy.zeros(n_parent_bits, dtype = numpy.uint8)
	child2_bits = numpy.zeros(n_parent_bits, dtype = numpy.uint8)

	j = 0
	for i in range(n_parent_bits):
		if parent_id_bits[i] == 1:
			if j < n_split_bits:
				if split_id_bits[j] == 1:
					child1_bits[i] = 1
				else:
					child2_bits[i] = 1
			else:
				child2_bits[i] = 1

			j += 1

	child1_bytes = numpy.packbits(child1_bits)
	child2_bytes = numpy.packbits(child2_bits)

	child1_id = child1_bytes.tostring().rstrip("\x00") # urgh C (null terminated strings)
	child2_id = child2_bytes.tostring().rstrip("\x00") # vs Python (not null terminated) strings

	return child1_id, child2_id

 def test_bad_count(self):
     packed0 = np.packbits(self.x, axis=0)
     assert_raises(ValueError, np.unpackbits, packed0, axis=0, count=-9)
     packed1 = np.packbits(self.x, axis=1)
     assert_raises(ValueError, np.unpackbits, packed1, axis=1, count=-9)
     packed = np.packbits(self.x)
     assert_raises(ValueError, np.unpackbits, packed, count=-57)

def write_data(dict_words):
    items = []
    with open(POSTS_FILE_PATH, 'r') as f:
        for post in ijson.items(f, 'item'):
            labels = [0] * len(CATEGORIES)
            was = False
            for hub in post['hubs']:
                for cur_label, label in enumerate(CATEGORIES):
                    if hub in label:
                        labels[cur_label] = 1
                        was = True

            if not was:
                continue

            words = [0] * BagOfWords.NUM_VOCABULARY_SIZE
            post_words = post['content'] + post['title']
            for word in post_words:
                if word in dict_words:
                    words[dict_words[word]] = 1
                else:
                    words[BagOfWords.NUM_VOCABULARY_SIZE - 1] = 1

            labels = np.packbits(labels).tolist()
            words = np.packbits(words).tolist()
            items.append((labels, words))

    shuffle(items)
    train_set_size = int(len(items) * BagOfWords.TRAIN_RATIO)
    _write_set(TF_ONE_SHOT_TRAIN_FILE_PATH, items[:train_set_size])
    _write_set(TF_ONE_SHOT_EVAL_FILE_PATH, items[train_set_size:])

    print('Set size : ', len(items))
    print('Train set size : ', train_set_size)
    print('Eval set size : ', len(items) - train_set_size)

    def payloadExists(self):
        buffer = []
        if self.color_flag:
            row = self.img[0]
            stop_flag = 0
            for col in row:
                if col[0] & numpy.uint8(1) == 1:
                    buffer.append(1)
                else:
                    buffer.append(0)

                if stop_flag == 7:
                    print(buffer)
                    valid = chr(numpy.packbits(buffer)[0])
                    if valid == "<":
                        return True
                    else:
                        return False
                stop_flag += 1
        else:
            i = 0
            row = self.img[0]
            while i < 8:
                if row[i] & numpy.uint8(1) == 1:
                    buffer.append(1)
                else:
                    buffer.append(0)
                i+= 1
            valid = chr(numpy.packbits(buffer)[0])
            if valid == "<":
                return True
            else:
                return False

    def image_from_bits(self, bits, filename):
        # Convert the received payload to an image and save it
        # No return value required .
        pixel_values = np.array([], dtype=np.uint)

        #Retrieve length of each row
        row_length_bits = bits[0:8]
        row_length = np.packbits(row_length_bits)[0]

        #print "Forming image from bits..."
        #print "Row length bits: " + str(row_length_bits)
        #print "\tFound row length of " + str(row_length)
        #print

        pixel_idx = 8
        while pixel_idx < len(bits):

            pixel_bits = bits[pixel_idx:pixel_idx+8]
            pixel_value = np.packbits(pixel_bits)[0]
            pixel_values = np.append(pixel_values, pixel_value)

            pixel_idx += 8 #int

        img = Image.new('L', (len(pixel_values) / row_length, row_length)) 
        img.putdata(pixel_values)
        img.save(filename)

 def extractPayload(self):
     if self.payloadExists() is False:
         raise Exception("Error: carrier does not contain a payload")
     xml = ""
     shape = self.img.shape
     payload_bits = np.copy(self.img)
     payload_bits &= 1
     if len(shape) > 2:
         red = payload_bits[:, :, 0]
         green = payload_bits[:, :, 1]
         blue = payload_bits[:, :, 2]
         payload_buff = np.concatenate((red.flatten('C'), green.flatten('C'), blue.flatten('C')), axis=0)
         payload = np.packbits(payload_buff)
         payload_list = [chr(item) for item in payload]
         for item in payload_list:
             xml += item
             if item == '>':
                 termination = re.search(r"</payload>", xml)
                 if termination:
                     final_payload = Payload(None, -1, xml)
                     return final_payload
     else:
         payload = np.packbits(payload_bits)
         payload_list = [chr(item) for item in payload]
         for item in payload_list:
             xml += item
             if item == '>':
                 termination = re.search(r"</payload>", xml)
                 if termination:
                     final_payload = Payload(None, -1, xml)
                     return final_payload

def test_unpackbits_count():
    # test complete invertibility of packbits and unpackbits with count
    x = np.array([
        [1, 0, 1, 0, 0, 1, 0],
        [0, 1, 1, 1, 0, 0, 0],
        [0, 0, 1, 0, 0, 1, 1],
        [1, 1, 0, 0, 0, 1, 1],
        [1, 0, 1, 0, 1, 0, 1],
        [0, 0, 1, 1, 1, 0, 0],
        [0, 1, 0, 1, 0, 1, 0],
    ], dtype=np.uint8)

    padded1 = np.zeros(57, dtype=np.uint8)
    padded1[:49] = x.ravel()

    packed = np.packbits(x)
    for count in range(58):
        unpacked = np.unpackbits(packed, count=count)
        assert_equal(unpacked.dtype, np.uint8)
        assert_array_equal(unpacked, padded1[:count])
    for count in range(-1, -57, -1):
        unpacked = np.unpackbits(packed, count=count)
        assert_equal(unpacked.dtype, np.uint8)
        # count -1 because padded1 has 57 instead of 56 elements
        assert_array_equal(unpacked, padded1[:count-1])
    for kwargs in [{}, {'count': None}]:
        unpacked = np.unpackbits(packed, **kwargs)
        assert_equal(unpacked.dtype, np.uint8)
        assert_array_equal(unpacked, padded1[:-1])
    assert_raises(ValueError, np.unpackbits, packed, count=-57)

    padded2 = np.zeros((9, 9), dtype=np.uint8)
    padded2[:7, :7] = x

    packed0 = np.packbits(x, axis=0)
    packed1 = np.packbits(x, axis=1)
    for count in range(10):
        unpacked0 = np.unpackbits(packed0, axis=0, count=count)
        assert_equal(unpacked0.dtype, np.uint8)
        assert_array_equal(unpacked0, padded2[:count, :x.shape[1]])
        unpacked1 = np.unpackbits(packed1, axis=1, count=count)
        assert_equal(unpacked1.dtype, np.uint8)
        assert_array_equal(unpacked1, padded2[:x.shape[1], :count])
    for count in range(-1, -9, -1):
        unpacked0 = np.unpackbits(packed0, axis=0, count=count)
        assert_equal(unpacked0.dtype, np.uint8)
        # count -1 because one extra zero of padding
        assert_array_equal(unpacked0, padded2[:count-1, :x.shape[1]])
        unpacked1 = np.unpackbits(packed1, axis=1, count=count)
        assert_equal(unpacked1.dtype, np.uint8)
        assert_array_equal(unpacked1, padded2[:x.shape[0], :count-1])
    for kwargs in [{}, {'count': None}]:
        unpacked0 = np.unpackbits(packed0, axis=0, **kwargs)
        assert_equal(unpacked0.dtype, np.uint8)
        assert_array_equal(unpacked0, padded2[:-1, :x.shape[1]])
        unpacked1 = np.unpackbits(packed1, axis=1, **kwargs)
        assert_equal(unpacked1.dtype, np.uint8)
        assert_array_equal(unpacked1, padded2[:x.shape[0], :-1])
    assert_raises(ValueError, np.unpackbits, packed0, axis=0, count=-9)
    assert_raises(ValueError, np.unpackbits, packed1, axis=1, count=-9)

def calculate_node_hashes(children_a, children_b, taxon_order):
	n_taxa = len(taxon_order)
	children = set.union(children_a, children_b)

	parent_boolean = numpy.zeros(n_taxa, dtype=numpy.uint8)
	split_boolean = numpy.zeros(n_taxa, dtype=numpy.uint8)

	i = 0
	for j in range(n_taxa):
		t = taxon_order[j]
		if t in children:
			parent_boolean[j] = 1

			if i == 0:
				if t in children_a:
					a_first = True
				else:
					a_first = False

			if (t in children_b) ^ a_first: # first child always "True"
				split_boolean[i] = 1

			i += 1

	parent_packed = numpy.packbits(parent_boolean)
	split_packed = numpy.packbits(split_boolean)

	parent_id = parent_packed.tostring()
	split_id = split_packed.tostring()

	return parent_id, split_id

 def write(self, filename):
     header_bytes = struct.pack(CHUNK_HEADER_FORMAT, self.data_size, self.board_size, self.input_planes, self.is_test)
     position_bytes = np.packbits(self.pos_features).tostring()
     next_move_bytes = np.packbits(self.next_moves).tostring()
     with gzip.open(filename, "wb", compresslevel=6) as f:
         f.write(header_bytes)
         f.write(position_bytes)
         f.write(next_move_bytes)

def convert(data, se):
    """Convert data according to the schema encoding"""
    dtype = data.dtype
    type = se.type
    converted_type = se.converted_type
    if dtype.name in typemap:
        if type in revmap:
            out = data.values.astype(revmap[type], copy=False)
        elif type == parquet_thrift.Type.BOOLEAN:
            padded = np.lib.pad(data.values, (0, 8 - (len(data) % 8)),
                                'constant', constant_values=(0, 0))
            out = np.packbits(padded.reshape(-1, 8)[:, ::-1].ravel())
        elif dtype.name in typemap:
            out = data.values
    elif "S" in str(dtype)[:2] or "U" in str(dtype)[:2]:
        out = data.values
    elif dtype == "O":
        try:
            if converted_type == parquet_thrift.ConvertedType.UTF8:
                out = array_encode_utf8(data)
            elif converted_type is None:
                if type in revmap:
                    out = data.values.astype(revmap[type], copy=False)
                elif type == parquet_thrift.Type.BOOLEAN:
                    padded = np.lib.pad(data.values, (0, 8 - (len(data) % 8)),
                                        'constant', constant_values=(0, 0))
                    out = np.packbits(padded.reshape(-1, 8)[:, ::-1].ravel())
                else:
                    out = data.values
            elif converted_type == parquet_thrift.ConvertedType.JSON:
                out = np.array([json.dumps(x).encode('utf8') for x in data],
                               dtype="O")
            elif converted_type == parquet_thrift.ConvertedType.BSON:
                out = data.map(tobson).values
            if type == parquet_thrift.Type.FIXED_LEN_BYTE_ARRAY:
                out = out.astype('S%i' % se.type_length)
        except Exception as e:
            ct = parquet_thrift.ConvertedType._VALUES_TO_NAMES[
                converted_type] if converted_type is not None else None
            raise ValueError('Error converting column "%s" to bytes using '
                             'encoding %s. Original error: '
                             '%s' % (data.name, ct, e))
    elif converted_type == parquet_thrift.ConvertedType.TIMESTAMP_MICROS:
        out = np.empty(len(data), 'int64')
        time_shift(data.values.view('int64'), out)
    elif converted_type == parquet_thrift.ConvertedType.TIME_MICROS:
        out = np.empty(len(data), 'int64')
        time_shift(data.values.view('int64'), out)
    elif type == parquet_thrift.Type.INT96 and dtype.kind == 'M':
        ns_per_day = (24 * 3600 * 1000000000)
        day = data.values.view('int64') // ns_per_day + 2440588
        ns = (data.values.view('int64') % ns_per_day)# - ns_per_day // 2
        out = np.empty(len(data), dtype=[('ns', 'i8'), ('day', 'i4')])
        out['ns'] = ns
        out['day'] = day
    else:
        raise ValueError("Don't know how to convert data type: %s" % dtype)
    return out

def test_unpackbits_large():
    # test all possible numbers via comparison to already tested packbits
    d = np.arange(277, dtype=np.uint8)
    assert_array_equal(np.packbits(np.unpackbits(d)), d)
    assert_array_equal(np.packbits(np.unpackbits(d[::2])), d[::2])
    d = np.tile(d, (3, 1))
    assert_array_equal(np.packbits(np.unpackbits(d, axis=1), axis=1), d)
    d = d.T.copy()
    assert_array_equal(np.packbits(np.unpackbits(d, axis=0), axis=0), d)

    def _generate_masks(self):
        """Creates left and right masks for all hash lengths."""
        tri_size = MAX_HASH_SIZE + 1
        # Called once on fitting, output is independent of hashes
        left_mask = np.tril(np.ones((tri_size, tri_size), dtype=int))[:, 1:]
        right_mask = left_mask[::-1, ::-1]

        self._left_mask = np.packbits(left_mask).view(dtype=HASH_DTYPE)
        self._right_mask = np.packbits(right_mask).view(dtype=HASH_DTYPE)

def main(args):
    x1 = np.load(args.infile1)
    x2 = np.load(args.infile2)
    assert len(x1.shape) == 2, 'infile1 should be 2d array!'
    assert len(x2.shape) == 2, 'infile2 should be 2d array!'
    assert x1.shape[0] == x2.shape[0], 'two infile should have same rows!'
    x1 = np.unpackbits(x1, axis=1)
    x2 = np.unpackbits(x2, axis=1)
    r1 = x1.shape[1] if args.row1 == 0 else args.row1
    r2 = x2.shape[1] if args.row2 == 0 else args.row2
    N = x1.shape[0]
    print(r1, r2, N)
    x1 = np.packbits(x1[:, :r1].T, axis=1)
    x2 = np.packbits(x2[:, :r2].T, axis=1)

    if args.gpu >= 0:
        chainer.cuda.get_device(args.gpu).use()
        x1 = cuda.to_gpu(x1)
        x2 = cuda.to_gpu(x2)
        xp = cupy
    else:
        xp = np
    # popcount LUT
    pc = xp.zeros(256, dtype=np.uint8)
    for i in range(256):
        pc[i] = ( i & 1 ) + pc[i/2]

    hamm = xp.zeros((r1, r2), dtype=np.int32)
    for i in tqdm(range(r1)):
        x1i = xp.tile(x1[i], (r2, 1))
        if args.operation == 'xor':
            hamm[i] = xp.take(pc, xp.bitwise_xor(x1i, x2).astype(np.int32)).sum(axis=1)
        elif args.operation == 'nand':
            hamm[i] = xp.take(pc, xp.invert(xp.bitwise_and(x1i, x2)).astype(np.int32)).sum(axis=1)
        #for j in range(r2):
            #hamm[i, j] = xp.take(pc, xp.bitwise_xor(x1[i], x2[j])).sum()
    x1non0 = xp.tile((x1.sum(axis=1)>0), (r2, 1)).T.astype(np.int32)
    x2non0 = xp.tile((x2.sum(axis=1)>0), (r1, 1)).astype(np.int32)
    print(x1non0.shape, x2non0.shape)
    non0filter = x1non0 * x2non0
    print(non0filter.max(), non0filter.min())
    hamm = non0filter * hamm + np.iinfo(np.int32).max * (1 - non0filter)
    #non0filter *= np.iinfo(np.int32).max
    #hamm *= non0filter
    if xp == cupy:
        hamm = hamm.get()
    #xp.savetxt(args.out, hamm, delimiter=args.delim)
    np.save(args.out, hamm)

    if args.nearest > 0:
        hamm_s = np.sort(hamm.flatten())
        hamm_as = np.argsort(hamm.flatten())
        x, y = np.unravel_index(hamm_as[:args.nearest], hamm.shape)
        fname, ext = os.path.splitext(args.out)
        np.savetxt(fname + '_top{0}.tsv'.format(args.nearest),
            np.concatenate((x[np.newaxis], y[np.newaxis], hamm_s[np.newaxis,:args.nearest]), axis=0).T,
            fmt='%d', delimiter='\t')

    def updateState(self):
	# TO DO: update the state variables reflected to vehicle data
	# i.e. to maintain continuity if vehicle data reread is requested
	# you need to map all the state control here
	setBigEndiNumberToNpArr(self.vehicle2_unpacked, getArrayIdxFromStartBit(55), 1,int(self.radar_poweron))
	self.vehicle2.data = np.packbits(self.vehicle2_unpacked).tolist()
	setBigEndiNumberToNpArr(self.vehicle2_unpacked, getArrayIdxFromStartBit(22), 1,int(self.clear_fault_on))
	self.vehicle2.data = np.packbits(self.vehicle2_unpacked).tolist()
	setBigEndiNumberToNpArr(self.vehicle2_unpacked, getArrayIdxFromStartBit(56), 1,int(self.rawdata_on))
	self.vehicle2.data = np.packbits(self.vehicle2_unpacked).tolist()

def parameter_from_population(individual):
    first_gene = individual[:8]
    second_gene = individual[8:16]
    third_gene = individual[16:]
    first_gene = np.packbits(first_gene, axis=-1)
    second_gene = np.packbits(second_gene, axis=-1)
    third_gene = np.packbits(third_gene, axis=-1)
    param_mutation = first_gene * coef_mut
    param_cross = second_gene * coef_cross
    param_population_size = int(third_gene * coef_population_size)
    return param_cross[0], param_mutation[0], param_population_size

def test_pack_unpack_order():
    a = np.array([[2], [7], [23]], dtype=np.uint8)
    b = np.unpackbits(a, axis=1)
    assert_equal(b.dtype, np.uint8)
    b_little = np.unpackbits(a, axis=1, bitorder='little')
    b_big = np.unpackbits(a, axis=1, bitorder='big')
    assert_array_equal(b, b_big)
    assert_array_equal(a, np.packbits(b_little, axis=1, bitorder='little'))
    assert_array_equal(b[:,::-1], b_little)
    assert_array_equal(a, np.packbits(b_big, axis=1, bitorder='big'))
    assert_raises(ValueError, np.unpackbits, a, bitorder='r')
    assert_raises(TypeError, np.unpackbits, a, bitorder=10)

def gene_to_noise_params(individual, display=False):
    first_gene = individual[:gene_size]
    second_gene = individual[gene_size:gene_size * 2]
    third_gene = individual[gene_size * 2:]
    if display:
        print first_gene, second_gene, third_gene
    first_gene = np.packbits(first_gene, axis=-1)
    second_gene = np.packbits(second_gene, axis=-1)
    third_gene = np.packbits(third_gene, axis=-1)
    noise_amp = first_gene * coef_amp
    noise_freq_row = second_gene * coef_freq
    noise_freq_col = third_gene * coef_freq
    return noise_amp[0], noise_freq_row[0], noise_freq_col[0]