def extractFile(filename, start, end, palette, names):
    dims = []

    with open(filename, 'rb') as artFile:
        artFile.seek(16)

        count = end - start + 1
        widths = [x[0] for x in struct.iter_unpack('<H', artFile.read(count * 2))]
        if len(widths) < count:
            raise BadArtFileException("Couldn't read enough widths.")
        heights = [x[0] for x in struct.iter_unpack('<H', artFile.read(count * 2))]
        if len(heights) < count:
            raise BadArtFileException("Couldn't read enough heights.")
        dims = zip(widths, heights)

        #don't need the rest of this
        artFile.seek(count * 4, 1)

        for dim in enumerate(dims):
            width, height = dim[1]
            if width == 0 or height == 0:
                continue
            data = artFile.read(width * height)
            if len(data) < width * height:
                raise BadArtFileException("Not enough data.")
            data = rearrangeData(data, width, height)

            name = ""
            try:
                name = names[start + dim[0]]
            except KeyError:
                name = "TILE{:04d}".format(start + dim[0])

            print(" -> Writing out {:s}...".format(name))
            writeBMP(name, width, height, palette, data)

def readPlanes(hdr, data):
    planedata = (x[0] for x in iter_unpack("24s", data[hdr.planeDataOffset:][:hdr.planeCount * 24]))
    normals   = (Point(*x) for x in iter_unpack("<3i", data[hdr.normalListOffset:][:hdr.planeCount * 24]))
    data = BytesIO(data[hdr.planeListOffset:])
    for plane, normal in zip(planedata, normals):
        planePointCount, unknown1, texture, unknown2 = unpack("<2BH6s", data.read(10))
        planePoints = [PlanePoint(int(point[0]/12), *point[1:])
            for point in iter_unpack("<IHH", data.read(planePointCount * 8))]
        yield Plane(unknown1, texture, unknown2, planePoints, normal, plane)

def packets_from_file(filename, recording_time, chunksize=102):
    with open(filename, "rb") as f:
        while True:
            chunk = f.read(chunksize)
            if chunk:
                sensor_time = datetime.timedelta(milliseconds=struct.unpack("<I", chunk[:4])[0])  # ms since start of capturing system
                fused_time = recording_time + sensor_time
                packet = {
                    'timestamp': fused_time,
                    'data': {}
                }
                counter = 0
                for qw, qx, qy, qz, ax, ay, az in struct.iter_unpack("<hhhhhhh", chunk[4:]):
                    scale_q = (1.0 / (1 << 14))
                    scale_a = 1.0 / 100
                    packet['data'][sensor_id[counter]] = {
                        # Decode Quaternion: Orientation data (unit-less)
                        'orientation': [round(scale_q * qw, 6),
                                        round(scale_q * qx, 6),
                                        round(scale_q * qy, 6),
                                        round(scale_q * qz, 6)],
                        # Decode Vector: Acceleration data (m/s^2)
                        'acceleration': [round(scale_a * ax, 2),
                                         round(scale_a * ay, 2),
                                         round(scale_a * az, 2)]
                    }
                    counter += 1
                yield packet
            else:
                break

def _decode_header(status_dict, names_list, data):

    names_list = [name[0].split(b"\x00", 1)[0] for name in 
        struct.iter_unpack(f"{FGC_MAX_DEV_LEN + 1}s", 
                            data[0 : FGCD_MAX_DEVS * (FGC_MAX_DEV_LEN + 1)])]

    offset = FGCD_MAX_DEVS * (FGC_MAX_DEV_LEN + 1)

    (status_dict["hostname"], 
        status_dict["recv_time_sec"],
        status_dict["recv_time_usec"], 
        status_dict["id"],
        status_dict["sequence"],
        status_dict["send_time_sec"],
        status_dict["send_time_usec"],
        status_dict["time_sec"],
        status_dict["time_usec"]) = struct.unpack(f"!{HOST_NAME_MAX}sllllllll", 
                                                    data[offset:])

    status_dict["hostname"] = status_dict["hostname"].split(b"\x00", 1)[0].decode()

    if not status_dict["hostname"]:
        status_dict.clear()
        return None

    return status_dict, names_list

    def give_motion_frame(self):
        """Read the next sample from the POS file and assign to coil objects.
        :returns: error code, packed bytestring with dataframe response
        """
        if self.motion_lines_read < self.max_num_frames:

            measurements = self.file.read(self.bytes_in_frame)

            # there appears to be no timestamp in the data
            timestamp = MocapParent.timestamp_to_microsecs(self.motion_lines_read * self.frame_time)

            # 7I is integers for: x, y, z, phi, theta, rms, extra
            measurements_by_coil = struct.iter_unpack('< 7f', measurements[:])
            #print('&&&',measurements_by_coil)
            for coilvals, coilobj in zip(measurements_by_coil, self.component.coils):
                floatvals = MocapParent.make_float_or_zero(coilvals)
                coilobj.set_args_to_vars(floatvals, mapping=self.mappings)

            self.component.timestamp = timestamp
            self.latest_timestamp = timestamp
            print('test timestamp!!', self.latest_timestamp)

            self.motion_lines_read += 1
            return 3, DataFrame(components=[self.component]).pack_components_to_df(), None

        # no more data available (static, give up)
        else:
            print("All the frames have been read")
            return 4, "No more frames left in mocap file"

def parse_dict(data):
    dct = {}
    mode = 'pad'
    key = []
    value = []

    for c in struct.iter_unpack('c', data):
        c = c[0]

        if mode == 'pad':
            if c in (bytes([0]), bytes([3])):
                continue
            else:
                mode = 'key'

        if mode == 'key':
            if c == bytes([0]):
                mode = 'value'
            else:
                key.append(c.decode())

        elif mode == 'value':
            if c == bytes([0]):
                dct[''.join(key)] = ''.join(value)
                key = []
                value = []
                mode = 'pad'
            else:
                value.append(c.decode())

    return dct

 def test_module_func(self):
     # Sanity check for the global struct.iter_unpack()
     it = struct.iter_unpack('>IB', bytes(range(1, 11)))
     self.assertEqual(next(it), (0x01020304, 5))
     self.assertEqual(next(it), (0x06070809, 10))
     self.assertRaises(StopIteration, next, it)
     self.assertRaises(StopIteration, next, it)

    def parse_minor_mesh_form(self, mesh_form, lod_lev=0):

        if lod_lev not in self.lods:
            self.lods[lod_lev] = {}
            self.lods[lod_lev]["mats"] = []
            self.lods[lod_lev]["altmats"] = []
            self.lods[lod_lev]["lightflags"] = []

        vers_form = self.iff.read_data()
        mesh_vers = int(vers_form["name"].decode("ascii"))
        self.lods[lod_lev]["version"] = mesh_vers
        mnrmsh_read = 16  # Bytes for version form header

        while mnrmsh_read < mesh_form["length"]:
            mdat = self.iff.read_data()

            mnrmsh_read += 8 + mdat["length"]
            mnrmsh_read += 1 if mdat["length"] % 2 == 1 else 0

            if mdat["name"] == b"NAME":
                self.lods[lod_lev]["name"] = (
                    self.parse_cstr(mdat["data"], 0))
            elif mdat["name"] == b"FACE":

                # This isn't part of Wing Blender, so I'm using features
                # from newer versions of Python 3.
                for f in struct.iter_unpack(self.FACE_FMT, mdat["data"]):
                    if f[2] not in self.lods[lod_lev]["mats"]:
                        self.lods[lod_lev]["mats"].append(f[2])
                    if f[5] not in self.lods[lod_lev]["lightflags"]:
                        self.lods[lod_lev]["lightflags"].append(f[5])
                    if f[6] not in self.lods[lod_lev]["altmats"]:
                        self.lods[lod_lev]["altmats"].append(f[6])

def parse_compact_node6_info(data):
    """
    Unpack a list of node id, IPv6 address and port returned by a find_node or get_peers request.
    """
    for frame in struct.iter_unpack("!20s8HH", data):
        # (id, (ipv6, port))
        yield {'id':frame[0], 'addr':('{:04X}:{:04X}:{:04X}:{:04X}:{:04X}:{:04X}:{:04X}:{:04X}'.format(*frame[1:9]), frame[9])}

def get_intlist(elf):
    intdata = elf.get_section_by_name("intList").data()

    header_sz = struct.calcsize(intlist_header_fmt)
    header = struct.unpack_from(intlist_header_fmt, intdata, 0)
    intdata = intdata[header_sz:]

    spurious_code = header[0]
    spurious_nocode = header[1]

    debug("spurious handler (code)    : %s" % hex(header[0]))
    debug("spurious handler (no code) : %s" % hex(header[1]))

    intlist = [i for i in
            struct.iter_unpack(intlist_entry_fmt, intdata)]

    debug("Configured interrupt routing")
    debug("handler    irq pri vec dpl")
    debug("--------------------------")

    for irq in intlist:
        debug("{0:<10} {1:<3} {2:<3} {3:<3} {4:<2}".format(
            hex(irq[0]),
            "-" if irq[1] == -1 else irq[1],
            "-" if irq[2] == -1 else irq[2],
            "-" if irq[3] == -1 else irq[3],
            irq[4]))

    return (spurious_code, spurious_nocode, intlist)

def decode_tilemap(tilemap, tiles, palettes):
    """
    Creates a list of colored tiles from a tilemap, raw tiles and a
    color palette
    """
    tilemap_tiles = []

    for char in struct.iter_unpack("<H", tilemap):
        char = char[0]
        # Layout: Section 9.3 at http://www.coranac.com/tonc/text/regbg.htm
        tile_id = char & ((1 << 10) - 1)
        flip_h = bool(char & (1 << 10))
        flip_v = bool(char & (1 << 11))
        pal_id = char >> 12

        pal = palettes[pal_id]
        tile = color_tile(tiles[tile_id], pal)
        tile_img = tile_image(tile)
        if flip_h:
            tile_img = tile_img.transpose(Image.FLIP_LEFT_RIGHT)
        if flip_v:
            tile_img = tile_img.transpose(Image.FLIP_TOP_BOTTOM)

        tilemap_tiles.append(tile_img)

    return tilemap_tiles

def parse_cmap(tree, node, data):
    i = -1
    for g, b, r in iter_unpack("3B", data):
        i += 1
        if r == g == b == 0:
            continue
        node.append(element("colour", text="%d %d %d" % (r, g, b), attrib={"id": str(i)}))

def parse_6int32(name, node, data):
    node.data = [Point(*x) for x in iter_unpack("<3i", data)]
    for point in node.data:
        corner = element("corner")
        for k, v in zip(point._fields, point):
            corner.append(element(k, text=str(v)))
        node.append(corner)

    def test_extract_palette(self):
        extracted = assets.files.backgrounds['mindscape'].extract_palette()

        with open(os.path.join(test_dir, 'mindscap_palette_extract_1.bin'), 'rb') as f:
            test_data = f.read()

        assert extracted == [i[0] for i in iter_unpack('<H', test_data)]

def parse_compact_node_info(data):
    """
    Unpack a list of node id, IPv4 address and port returned by a find_node or get_peers request.
    """
    for frame in struct.iter_unpack("!20s4BH", data):
        # (id, (ip, port))
        yield {'id':frame[0], 'addr':("{}.{}.{}.{}".format(frame[1], frame[2], frame[3], frame[4]), frame[5])}

def parse_htbl(tree, node, data):
    i = -1
    res = []
    for x in iter_unpack("256s", data):
        i += 1
        node.append(element("unknown", text=to_hex(x[0]), attrib={"id": str(i)}))
        res.append(x[0])
    node.data = res

def parse_hicl(tree, node, data):
    i = -1
    for x in iter_unpack(">H", data):
        i += 1
        r, g, b = unpack_15bpp(x[0])
        if r == g == b == 0:
            continue
        node.append(element("colour", text="%d %d %d" % (r, g, b), attrib={"id": str(i)}))

	def unpack_data(self, buf):

		unpacked_data = {}
		payload = struct.iter_unpack('iffffffff', buf[5:])
		for item in payload:
			if item != None:
				unpacked_data[item[0]] = item[1:]
		return unpacked_data

def unpack_data_from_file(file, fmt, offset=0):
    """Retrieve data from given file and unpack them according to specified format.

    file: The path name of the file or an opened file object
    fmt: Format specification
    offset: Offset from the start of the file from where to start the unpacking operation
    """
    if isinstance(file, str):
        if not os.path.isfile(file):
            raise ValueError('"{}" is not a regular file'.format(file))
    else:
        if not hasattr(file, 'read'):
            raise ValueError('Invalid file object')
        if hasattr(file, 'mode') and file.mode != 'rb':
            raise ValueError('Invalid opening mode')

    if not isinstance(offset, int) or offset < 0:
        raise ValueError('Invalid offset value')

    with fileutil.open_file(file, 'rb') as fp:
        file_len = fileutil.file_size(fp)
        pack_size = struct.calcsize(fmt)

        if file_len <= offset:
            return

        fp.seek(offset, os.SEEK_SET)

        if (file_len - offset) % pack_size == 0:
            if sys.version_info >= (3, 3):
                #return struct.iter_unpack(fmt, fp.read())  # Fix issue #1
                yield from struct.iter_unpack(fmt, fp.read())
            else:
                for unpacked_data in struct.iter_unpack(fmt, fp.read()):
                    yield unpacked_data
        else:
            # The length of the file isn't the multiple of struct.calcsize(fmt), so
            # don't call struct.iter_unpack directly.
            data = fp.read(pack_size)
            while data:
                if len(data) == pack_size:
                    yield struct.unpack(fmt, data)
                else:
                    break
                data = fp.read(pack_size)

def read_intlist(intlist_path):
    """read a binary file containing the contents of the kernel's .intList
    section. This is an instance of a header created by
    include/linker/intlist.ld:

     struct {
       u32_t num_vectors;       <- typically CONFIG_NUM_IRQS
       struct _isr_list isrs[]; <- Usually of smaller size than num_vectors
    }

    Followed by instances of struct _isr_list created by IRQ_CONNECT()
    calls:

    struct _isr_list {
        /** IRQ line number */
        s32_t irq;
        /** Flags for this IRQ, see ISR_FLAG_* definitions */
        s32_t flags;
        /** ISR to call */
        void *func;
        /** Parameter for non-direct IRQs */
        void *param;
    };
    """

    intlist = {}

    prefix = endian_prefix()

    intlist_header_fmt = prefix + "II"
    intlist_entry_fmt = prefix + "iiII"

    with open(intlist_path, "rb") as fp:
        intdata = fp.read()

    header_sz = struct.calcsize(intlist_header_fmt)
    header = struct.unpack_from(intlist_header_fmt, intdata, 0)
    intdata = intdata[header_sz:]

    debug(str(header))

    intlist["num_vectors"]    = header[0]
    intlist["offset"]         = header[1]

    intlist["interrupts"] = [i for i in
            struct.iter_unpack(intlist_entry_fmt, intdata)]

    debug("Configured interrupt routing")
    debug("handler    irq flags param")
    debug("--------------------------")

    for irq in intlist["interrupts"]:
        debug("{0:<10} {1:<3} {2:<3}   {3}".format(
            hex(irq[2]), irq[0], irq[1], hex(irq[3])))

    return intlist