def save_nparray_to_hdfs(fname, X, hdfs):
	'''
	An instance of numpy's savetext function to enable saving numpy 
	arrays in HDFS as text files
	'''
	fmt = '%.18e'
	delimiter = ' '
	newline = '\n'
	if isinstance(fmt, bytes):
		fmt = asstr(fmt)
	delimiter = asstr(delimiter)
	X = np.asarray(X)
	if X.ndim == 1:
		if X.dtype.names is None:
			X = np.atleast_2d(X).T
			ncol = 1
		else:
			ncol = len(X.dtype.descr)
	else:
		ncol = X.shape[1]
	n_fmt_chars = fmt.count('%')
	fmt = [fmt, ] * ncol
	format = delimiter.join(fmt)
	first = True
	for row in X:
		if first:
			hdfs.create_file(fname, asbytes(format % tuple(row) + newline), overwrite=True)
			first = False
		else:	
			hdfs.appendfile(fname, asbytes(format % tuple(row) + newline))

def _read_metadata(bitmap):
    metadata = {}
    models = [(name[5:], number) for name, number in
        METADATA_MODELS.__dict__.items() if name.startswith('FIMD_')]

    tag = ctypes.c_void_p()
    for model_name, number in models:
        mdhandle = _FI.FreeImage_FindFirstMetadata(number, bitmap,
                                                   ctypes.byref(tag))
        mdhandle = ctypes.c_void_p(mdhandle)
        if mdhandle:
            more = True
            while more:
                tag_name = asstr(_FI.FreeImage_GetTagKey(tag))
                tag_type = _FI.FreeImage_GetTagType(tag)
                byte_size = _FI.FreeImage_GetTagLength(tag)
                char_ptr = ctypes.c_char * byte_size
                tag_str = char_ptr.from_address(_FI.FreeImage_GetTagValue(tag))
                if tag_type == METADATA_DATATYPE.FIDT_ASCII:
                    tag_val = asstr(tag_str.value)
                else:
                    tag_val = numpy.fromstring(tag_str,
                            dtype=METADATA_DATATYPE.dtypes[tag_type])
                    if len(tag_val) == 1:
                        tag_val = tag_val[0]
                metadata[(model_name, tag_name)] = tag_val
                more = _FI.FreeImage_FindNextMetadata(mdhandle, ctypes.byref(tag))
            _FI.FreeImage_FindCloseMetadata(mdhandle)
    return metadata

    def _read_var(self):
        name = asstr(self._unpack_string())
        dimensions = []
        shape = []
        dims = self._unpack_int()

        for i in range(dims):
            dimid = self._unpack_int()
            dimname = self._dims[dimid]
            dimensions.append(dimname)
            dim = self.dimensions[dimname]
            shape.append(dim)
        dimensions = tuple(dimensions)
        shape = tuple(shape)

        attributes = self._read_att_array()
        nc_type = self.fp.read(4)
        vsize = self._unpack_int()
        begin = [self._unpack_int, self._unpack_int64][self.version_byte-1]()

        typecode, size = TYPEMAP[nc_type]
        if typecode is 'c':
            dtype_ = '>c'
        else:
            dtype_ = '>%s' % typecode
            if size > 1: dtype_ += str(size)

        return name, dimensions, shape, attributes, typecode, size, dtype_, begin, vsize

    def get_variables(self, variable_names=None):
        ''' get variables from stream as dictionary

        variable_names   - optional list of variable names to get

        If variable_names is None, then get all variables in file
        '''
        if isinstance(variable_names, basestring):
            variable_names = [variable_names]
        self.mat_stream.seek(0)
        # set up variable reader
        self.initialize_read()
        mdict = {}
        while not self.end_of_stream():
            hdr, next_position = self.read_var_header()
            name = asstr(hdr.name)
            if variable_names and name not in variable_names:
                self.mat_stream.seek(next_position)
                continue
            mdict[name] = self.read_var_array(hdr)
            self.mat_stream.seek(next_position)
            if variable_names:
                variable_names.remove(name)
                if len(variable_names) == 0:
                    break
        return mdict

    def get_variables(self, variable_names=None):
        ''' get variables from stream as dictionary

        Parameters
        ----------
        variable_names : None or str or sequence of str, optional
            variable name, or sequence of variable names to get from Mat file /
            file stream.  If None, then get all variables in file
        '''
        if isinstance(variable_names, string_types):
            variable_names = [variable_names]
        elif variable_names is not None:
            variable_names = list(variable_names)
        self.mat_stream.seek(0)
        # set up variable reader
        self.initialize_read()
        mdict = {}
        while not self.end_of_stream():
            hdr, next_position = self.read_var_header()
            name = asstr(hdr.name)
            if variable_names is not None and name not in variable_names:
                self.mat_stream.seek(next_position)
                continue
            mdict[name] = self.read_var_array(hdr)
            self.mat_stream.seek(next_position)
            if variable_names is not None:
                variable_names.remove(name)
                if len(variable_names) == 0:
                    break
        return mdict

    def info(self, source):
        source, close_it = self._open(source)

        try:

            # read and validate header line
            line = source.readline()
            mmid, matrix, format, field, symmetry  = \
              [asstr(part.strip().lower()) for part in line.split()]
            if not mmid.startswith('%%matrixmarket'):
                raise ValueError,'source is not in Matrix Market format'

            assert matrix == 'matrix',`line`

            # ??? Is this necessary?  I don't see 'dense' or 'sparse' in the spec
            # http://math.nist.gov/MatrixMarket/formats.html
            if format == 'dense': format = self.FORMAT_ARRAY
            elif format == 'sparse': format = self.FORMAT_COORDINATE

            # skip comments
            while line.startswith(asbytes('%')): line = source.readline()

            line = line.split()
            if format == self.FORMAT_ARRAY:
                assert len(line)==2,`line`
                rows,cols = map(float, line)
                entries = rows*cols
            else:
                assert len(line)==3,`line`
                rows, cols, entries = map(float, line)

            return (rows, cols, entries, format, field, symmetry)

        finally:
            if close_it: source.close()

def _filter_header(s):
    """Clean up 'L' in npz header ints.

    Cleans up the 'L' in strings representing integers. Needed to allow npz
    headers produced in Python2 to be read in Python3.

    Parameters
    ----------
    s : byte string
        Npy file header.

    Returns
    -------
    header : str
        Cleaned up header.

    """
    import tokenize

    if sys.version_info[0] >= 3:
        from io import StringIO
    else:
        from StringIO import StringIO

    tokens = []
    last_token_was_number = False
    for token in tokenize.generate_tokens(StringIO(asstr(s)).read):
        token_type = token[0]
        token_string = token[1]
        if last_token_was_number and token_type == tokenize.NAME and token_string == "L":
            continue
        else:
            tokens.append(token)
        last_token_was_number = token_type == tokenize.NUMBER
    return tokenize.untokenize(tokens)

def build_module(source_files, options=[], skip=[], only=[], module_name=None):
    """
    Compile and import a f2py module, built from the given files.

    """

    code = ("import sys; sys.path = %s; import numpy.f2py as f2py2e; "
            "f2py2e.main()" % repr(sys.path))

    d = get_module_dir()

    # Copy files
    dst_sources = []
    for fn in source_files:
        if not os.path.isfile(fn):
            raise RuntimeError("%s is not a file" % fn)
        dst = os.path.join(d, os.path.basename(fn))
        shutil.copyfile(fn, dst)
        dst_sources.append(dst)

        fn = os.path.join(os.path.dirname(fn), '.f2py_f2cmap')
        if os.path.isfile(fn):
            dst = os.path.join(d, os.path.basename(fn))
            if not os.path.isfile(dst):
                shutil.copyfile(fn, dst)

    # Prepare options
    if module_name is None:
        module_name = get_temp_module_name()
    f2py_opts = ['-c', '-m', module_name] + options + dst_sources
    if skip:
        f2py_opts += ['skip:'] + skip
    if only:
        f2py_opts += ['only:'] + only

    # Build
    cwd = os.getcwd()
    try:
        os.chdir(d)
        cmd = [sys.executable, '-c', code] + f2py_opts
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT)
        out, err = p.communicate()
        if p.returncode != 0:
            raise RuntimeError("Running f2py failed: %s\n%s"
                               % (cmd[4:], asstr(out)))
    finally:
        os.chdir(cwd)

        # Partial cleanup
        for fn in dst_sources:
            os.unlink(fn)

    # Import
    __import__(module_name)
    return sys.modules[module_name]

    def _read_att_array(self):
        header = self.fp.read(4)
        assert header in [ZERO, NC_ATTRIBUTE]
        count = self._unpack_int()

        attributes = {}
        for attr in range(count):
            name = asstr(self._unpack_string())
            attributes[name] = self._read_values()
        return attributes

    def _read_dim_array(self):
        header = self.fp.read(4)
        assert header in [ZERO, NC_DIMENSION]
        count = self._unpack_int()

        for dim in range(count):
            name = asstr(self._unpack_string())
            length = self._unpack_int() or None  # None for record dimension
            self.dimensions[name] = length
            self._dims.append(name)  # preserve order

def _read_string(f):
    """Read a string"""
    length = _read_long(f)
    if length > 0:
        chars = _read_bytes(f, length)
        _align_32(f)
        chars = asstr(chars)
    else:
        chars = None
    return chars

def _read_string(f):
    '''Read a string'''
    length = _read_long(f)
    if length > 0:
        chars = _read_bytes(f, length)
        _align_32(f)
        chars = asstr(chars)
    else:
        chars = ''
    return chars

    def _read_att_array(self):
        header = self.fp.read(4)
        if header not in [ZERO, NC_ATTRIBUTE]:
            raise ValueError("Unexpected header.")
        count = self._unpack_int()

        attributes = {}
        for attr in range(count):
            name = asstr(self._unpack_string())
            attributes[name] = self._read_values()
        return attributes

def _read_string(f):
    '''Read a string'''
    length = _read_long(f)
    if length > 0:
        chars = _read_bytes(f, length)
        _align_32(f)
        chars = asstr(chars)
    else:
        warnings.warn("warning: empty strings are now set to '' instead of None")
        chars = ''
    return chars

    def _read_dim_array(self):
        header = self.fp.read(4)
        if header not in [ZERO, NC_DIMENSION]:
            raise ValueError("Unexpected header.")
        count = self._unpack_int()

        for dim in range(count):
            name = asstr(self._unpack_string())
            length = self._unpack_int() or None  # None for record dimension
            self.dimensions[name] = length
            self._dims.append(name)  # preserve order

    def get_variables(self, variable_names=None):
        ''' get variables from stream as dictionary

        variable_names   - optional list of variable names to get

        If variable_names is None, then get all variables in file
        '''
        if isinstance(variable_names, string_types):
            variable_names = [variable_names]
        elif variable_names is not None:
            variable_names = list(variable_names)

        self.mat_stream.seek(0)
        # Here we pass all the parameters in self to the reading objects
        self.initialize_read()
        mdict = self.read_file_header()
        mdict['__globals__'] = []
        while not self.end_of_stream():
            hdr, next_position = self.read_var_header()
            name = asstr(hdr.name)
            if name in mdict:
                warnings.warn('Duplicate variable name "%s" in stream'
                              ' - replacing previous with new\n'
                              'Consider mio5.varmats_from_mat to split '
                              'file into single variable files' % name,
                              MatReadWarning, stacklevel=2)
            if name == '':
                # can only be a matlab 7 function workspace
                name = '__function_workspace__'
                # We want to keep this raw because mat_dtype processing
                # will break the format (uint8 as mxDOUBLE_CLASS)
                process = False
            else:
                process = True
            if variable_names and name not in variable_names:
                self.mat_stream.seek(next_position)
                continue
            try:
                res = self.read_var_array(hdr, process)
            except MatReadError as err:
                warnings.warn(
                    'Unreadable variable "%s", because "%s"' %
                    (name, err),
                    Warning, stacklevel=2)
                res = "Read error: %s" % err
            self.mat_stream.seek(next_position)
            mdict[name] = res
            if hdr.is_global:
                mdict['__globals__'].append(name)
            if variable_names:
                variable_names.remove(name)
                if len(variable_names) == 0:
                    break
        return mdict

    def list_variables(self):
        ''' list variables from stream '''
        self.mat_stream.seek(0)
        # set up variable reader
        self.initialize_read()
        vars = []
        while not self.end_of_stream():
            hdr, next_position = self.read_var_header()
            name = asstr(hdr.name)
            shape = self._matrix_reader.shape_from_header(hdr)
            info = mclass_info.get(hdr.mclass, 'unknown')
            vars.append((name, shape, info))

            self.mat_stream.seek(next_position)
        return vars

def read_minimat_vars(rdr):
    rdr.initialize_read()
    mdict = {'__globals__': []}
    i = 0
    while not rdr.end_of_stream():
        hdr, next_position = rdr.read_var_header()
        name = asstr(hdr.name)
        if name == '':
            name = 'var_%d' % i
            i += 1
        res = rdr.read_var_array(hdr, process=False)
        rdr.mat_stream.seek(next_position)
        mdict[name] = res
        if hdr.is_global:
            mdict['__globals__'].append(name)
    return mdict

    def list_variables(self):
        ''' list variables from stream '''
        self.mat_stream.seek(0)
        # Here we pass all the parameters in self to the reading objects
        self.initialize_read()
        self.read_file_header()
        vars = []
        while not self.end_of_stream():
            hdr, next_position = self.read_var_header()
            name = asstr(hdr.name)
            if name == '':
                # can only be a matlab 7 function workspace
                name = '__function_workspace__'

            shape = self._matrix_reader.shape_from_header(hdr)
            info = mclass_info.get(hdr.mclass, 'unknown')
            vars.append((name, shape, info))

            self.mat_stream.seek(next_position)
        return vars

    def info(self, source):
        source, close_it = self._open(source)

        try:

            # read and validate header line
            line = source.readline()
            mmid, matrix, format, field, symmetry = \
              [asstr(part.strip()) for part in line.split()]
            if not mmid.startswith('%%MatrixMarket'):
                raise ValueError('source is not in Matrix Market format')
            if not matrix.lower() == 'matrix':
                raise ValueError("Problem reading file header: " + line)

            # http://math.nist.gov/MatrixMarket/formats.html
            if format.lower() == 'array':
                format = self.FORMAT_ARRAY
            elif format.lower() == 'coordinate':
                format = self.FORMAT_COORDINATE

            # skip comments
            while line.startswith(b'%'):
                line = source.readline()

            line = line.split()
            if format == self.FORMAT_ARRAY:
                if not len(line) == 2:
                    raise ValueError("Header line not of length 2: " + line)
                rows, cols = map(int, line)
                entries = rows * cols
            else:
                if not len(line) == 3:
                    raise ValueError("Header line not of length 3: " + line)
                rows, cols, entries = map(int, line)

            return (rows, cols, entries, format, field.lower(), symmetry.lower())

        finally:
            if close_it:
                source.close()