def test_memmap_roundtrip():
    # Fixme: test crashes nose on windows.
    if not (sys.platform == 'win32' or sys.platform == 'cygwin'):
        for arr in basic_arrays + record_arrays:
            if arr.dtype.hasobject:
                # Skip these since they can't be mmap'ed.
                continue
            # Write it out normally and through mmap.
            nfn = os.path.join(tempdir, 'normal.npy')
            mfn = os.path.join(tempdir, 'memmap.npy')
            fp = open(nfn, 'wb')
            try:
                format.write_array(fp, arr)
            finally:
                fp.close()

            fortran_order = (
                arr.flags.f_contiguous and not arr.flags.c_contiguous)
            ma = format.open_memmap(mfn, mode='w+', dtype=arr.dtype,
                                    shape=arr.shape, fortran_order=fortran_order)
            ma[...] = arr
            del ma

            # Check that both of these files' contents are the same.
            fp = open(nfn, 'rb')
            normal_bytes = fp.read()
            fp.close()
            fp = open(mfn, 'rb')
            memmap_bytes = fp.read()
            fp.close()
            yield assert_equal_, normal_bytes, memmap_bytes

            # Check that reading the file using memmap works.
            ma = format.open_memmap(nfn, mode='r')
            del ma

def test_memmap_roundtrip():
    # XXX: test crashes nose on windows. Fix this
    if not (sys.platform == "win32" or sys.platform == "cygwin"):
        for arr in basic_arrays + record_arrays:
            if arr.dtype.hasobject:
                # Skip these since they can't be mmap'ed.
                continue
            # Write it out normally and through mmap.
            nfn = os.path.join(tempdir, "normal.npy")
            mfn = os.path.join(tempdir, "memmap.npy")
            fp = open(nfn, "wb")
            try:
                format.write_array(fp, arr)
            finally:
                fp.close()

            fortran_order = arr.flags.f_contiguous and not arr.flags.c_contiguous
            ma = format.open_memmap(mfn, mode="w+", dtype=arr.dtype, shape=arr.shape, fortran_order=fortran_order)
            ma[...] = arr
            del ma

            # Check that both of these files' contents are the same.
            fp = open(nfn, "rb")
            normal_bytes = fp.read()
            fp.close()
            fp = open(mfn, "rb")
            memmap_bytes = fp.read()
            fp.close()
            yield assert_equal, normal_bytes, memmap_bytes

            # Check that reading the file using memmap works.
            ma = format.open_memmap(nfn, mode="r")
            # yield assert_array_equal, ma, arr
            del ma

def test_version_2_0_memmap():
    # requires more than 2 byte for header
    dt = [(("%d" % i) * 100, float) for i in range(500)]
    d = np.ones(1000, dtype=dt)
    tf = tempfile.mktemp('', 'mmap', dir=tempdir)

    # 1.0 requested but data cannot be saved this way
    assert_raises(ValueError, format.open_memmap, tf, mode='w+', dtype=d.dtype,
                            shape=d.shape, version=(1, 0))

    ma = format.open_memmap(tf, mode='w+', dtype=d.dtype,
                            shape=d.shape, version=(2, 0))
    ma[...] = d
    del ma

    with warnings.catch_warnings(record=True) as w:
        warnings.filterwarnings('always', '', UserWarning)
        ma = format.open_memmap(tf, mode='w+', dtype=d.dtype,
                                shape=d.shape, version=None)
        assert_(w[0].category is UserWarning)
        ma[...] = d
        del ma

    ma = format.open_memmap(tf, mode='r')
    assert_array_equal(ma, d)

def gendata(
        data_path,
        label_path,
        data_out_path,
        label_out_path,
        num_person_in=5,  #observe the first 5 persons 
        num_person_out=2,  #then choose 2 persons with the highest score 
        max_frame=300):

    feeder = Feeder_kinetics(
        data_path=data_path,
        label_path=label_path,
        num_person_in=num_person_in,
        num_person_out=num_person_out,
        window_size=max_frame)

    sample_name = feeder.sample_name
    sample_label = []

    fp = open_memmap(
        data_out_path,
        dtype='float32',
        mode='w+',
        shape=(len(sample_name), 3, max_frame, 18, num_person_out))

    for i, s in enumerate(sample_name):
        data, label = feeder[i]
        print_toolbar(i * 1.0 / len(sample_name),
                      '({:>5}/{:<5}) Processing data: '.format(
                          i + 1, len(sample_name)))
        fp[i, :, 0:data.shape[1], :, :] = data
        sample_label.append(label)

    with open(label_out_path, 'wb') as f:
        pickle.dump((sample_name, list(sample_label)), f)

    def dump_electrode_data_circus(self, filename, chunks=1e9):
        self.load_mcs_data()
        itemsize = np.array([0.0], dtype=np.float32).nbytes
        data = self.electrodes_data
        n = len(next(iter(data.values())))  # num samples per channel
        n_items = int(chunks // itemsize)  # num chunked samples per chan
        total_n = sum(len(value) for value in data.values())  # num bytes total
        pbar = tqdm(
            total=total_n * itemsize, file=sys.stdout, unit_scale=1,
            unit='bytes')

        mmap_array = open_memmap(
            filename, mode='w+', dtype=np.float32, shape=(n, len(data)))

        names = sorted(data.keys(), key=lambda x: (x[0], int(x[1:])))
        for k, name in enumerate(names):
            value = data[name]
            offset, scale = self.get_electrode_offset_scale(name)
            i = 0
            n = len(value)

            while i * n_items < n:
                items = np.array(
                    value[i * n_items:min((i + 1) * n_items, n)])
                mmap_array[i * n_items:i * n_items + len(items), k] = \
                    (items - offset) * scale
                pbar.update(len(items) * itemsize)
                i += 1
        pbar.close()
        print('Channel order in "{}" is: {}'.format(filename, names))

    def setUp(self):
        self.data = sp.arange(80).reshape((2, 8, 5))

        self.memmap_data = npfor.open_memmap('temp.npy', mode='w+',
                                             shape=(2, 8, 5))

        self.memmap_data[:, :, :] = sp.arange(80).reshape(2, 8, 5)

 def _get_numpy_binary_array(self, name):
     """Return the an memmap object as represented by the .npy file"""
     filename = self._array_files.get(name)
     if filename is not None:
         return open_memmap(filename)
     else:
         return None

def gendata(data_path,
            out_path,
            ignored_sample_path=None,
            benchmark='xview',
            part='eval'):
    if ignored_sample_path != None:
        with open(ignored_sample_path, 'r') as f:
            ignored_samples = [
                line.strip() + '.skeleton' for line in f.readlines()
            ]
    else:
        ignored_samples = []
    sample_name = []
    sample_label = []
    for filename in os.listdir(data_path):
        if filename in ignored_samples:
            continue
        action_class = int(
            filename[filename.find('A') + 1:filename.find('A') + 4])
        subject_id = int(
            filename[filename.find('P') + 1:filename.find('P') + 4])
        camera_id = int(
            filename[filename.find('C') + 1:filename.find('C') + 4])

        if benchmark == 'xview':
            istraining = (camera_id in training_cameras)
        elif benchmark == 'xsub':
            istraining = (subject_id in training_subjects)
        else:
            raise ValueError()

        if part == 'train':
            issample = istraining
        elif part == 'val':
            issample = not (istraining)
        else:
            raise ValueError()

        if issample:
            sample_name.append(filename)
            sample_label.append(action_class - 1)

    with open('{}/{}_label.pkl'.format(out_path, part), 'wb') as f:
        pickle.dump((sample_name, list(sample_label)), f)
    # np.save('{}/{}_label.npy'.format(out_path, part), sample_label)

    fp = open_memmap(
        '{}/{}_data.npy'.format(out_path, part),
        dtype='float32',
        mode='w+',
        shape=(len(sample_label), 3, max_frame, num_joint, max_body))

    for i, s in enumerate(sample_name):
        print_toolbar(i * 1.0 / len(sample_label),
                      '({:>5}/{:<5}) Processing {:>5}-{:<5} data: '.format(
                          i + 1, len(sample_name), benchmark, part))
        data = read_xyz(
            os.path.join(data_path, s), max_body=max_body, num_joint=num_joint)
        fp[i, :, 0:data.shape[1], :, :] = data
    end_toolbar()

 def crop(self, item, focus, mode='loose', fixed=None, return_data=True):
     """Faster version of precomputed(item).crop(...)"""
     memmap = open_memmap(self.get_path(item), mode='r')
     swf = SlidingWindowFeature(memmap, self.sliding_window_)
     result = swf.crop(focus, mode=mode, fixed=fixed,
                       return_data=return_data)
     del memmap
     return result

def open_memmap(filename, mode='r+', dtype=None, shape=None,
                fortran_order=False, version=(1, 0), metafile=None):
    """Open a file and memory map it to an InfoMemmap object.

    This is similar to the numpy.lib.format.openmemmap() function but also
    deals with the meta data dictionary, which is read and written from a
    meta data file.

    The only extra argument over the numpy version is the meta data file name
    `metafile`.

    Parameters
    ----------
    metafile: str
        File name for which the `info` attribute of the returned InfoMemmap
        will be read from and written to. Default is None, where the it is
        assumed to be `filename` + ".meta".

    Returns
    -------
    marray: InfoMemmap
        The `info` is intialized as an empty dictionary if `mode` is 'w' or if
        the file corresponding to `metafile` does not exist.  The `metafile`
        attribute of marray is set to the `metafile` parameter unless `mode` is
        'r' or 'c' in which case it is set to None.
    """

    # Restrict to version (1,0) because we've only written write_header for
    # this version.
    if version != (1, 0):
        raise ValueError("Only version (1,0) is safe from this function.")

    # Memory map the data part.
    marray = npfor.open_memmap(filename, mode, dtype, shape, fortran_order,
                               version)

    # Get the file name for the meta data.
    if metafile is None:
        metafile = filename + '.meta'

    # Read the meta data if need be.
    if ('r' in mode or mode is 'c') and os.path.isfile(metafile):
        info_fid = open(metafile, 'r')
        try:
            infostring = info_fid.readline()
        finally:
            info_fid.close()
        info = safe_eval(infostring)
    else:
        info = {}

    # In read mode don't pass a metafile to protect the meta data.
    if mode is 'r' or mode is 'c':
        metafile = None

    marray = info_header.InfoMemmap(marray, info, metafile)

    return marray

    def create_empty(self, file_path=None, entries=1, field_names=None, data_types=None, memory_mode=False):
        """
        :param file_path: Optional. Full path for the output data file. If *memory_false* is 'false' and path is missing,
        then the file is created in the temp folder
        :param entries: Number of records in the dataset. Default is 1
        :param field_names: List of field names for this dataset. If no list is provided, the field 'data' will be created
        :param data_types: List of data types for the dataset. Types need to be NumPy data types (e.g. np.int16,
        np.float64). If no list of types are provided, type will be *np.float64*
        :param memory_mode: If true, dataset will be kept in memory. If false, the dataset will  be a memory-mapped numpy array
        :return: # nothing. Associates a dataset with the AequilibraEData object
        """

        if file_path is not None or memory_mode:
            if field_names is None:
                field_names = ['data']

            if data_types is None:
                data_types = [np.float64] * len(field_names)

            self.file_path = file_path
            self.entries = entries
            self.fields = field_names
            self.data_types = data_types
            self.aeq_index_type = np.uint64

            if memory_mode:
                self.memory_mode = MEMORY
            else:
                self.memory_mode = DISK
                if self.file_path is None:
                    self.file_path = self.random_name()

            # Consistency checks
            if not isinstance(self.fields, list):
                raise ValueError('Titles for fields, "field_names", needs to be a list')

            if not isinstance(self.data_types, list):
                raise ValueError('Data types, "data_types", needs to be a list')
            # The check below is not working properly with the QGIS importer
            # else:
            #     for dt in self.data_types:
            #         if not isinstance(dt, type):
            #             raise ValueError('Data types need to be Python or Numpy data types')

            for field in self.fields:
                if field in object.__dict__:
                    raise Exception(field + ' is a reserved name. You cannot use it as a field name')

            self.num_fields = len(self.fields)

            dtype = [('index', self.aeq_index_type)]
            dtype.extend([(self.fields[i], self.data_types[i]) for i in range(self.num_fields)])

            # the file
            if self.memory_mode:
                self.data = np.recarray((self.entries,), dtype=dtype)
            else:
                self.data = open_memmap(self.file_path, mode='w+', dtype=dtype, shape=(self.entries,))

 def setUp(self) :
     data = sp.arange(20)
     data.shape = (5,4)
     self.mat_arr = algebra.make_mat(data.copy(), axis_names=('ra', 'dec'))
     self.vect_arr = algebra.make_vect(data.copy(), axis_names=('ra', 'dec'))
     mem = npfor.open_memmap('temp.npy', mode='w+', shape=(5, 4))
     mem[:] = data
     self.vect_mem = algebra.make_vect(mem)
     self.arr = data.copy()

 def test_from_memmap(self) :
     # Works if constructed from array.
     data = npfor.open_memmap('temp.npy', mode='w+', shape=(4,3,3))
     data[:] = 5.0
     Mat = algebra.info_memmap(data, {'a': 'b'})
     Mat.flush()
     self.assertEqual(Mat.shape, (4, 3, 3))
     self.assertEqual(Mat.info['a'], 'b')
     self.assertTrue(sp.allclose(Mat, 5.0))
     self.assertTrue(isinstance(Mat,  sp.memmap))
     del Mat
     os.remove('temp.npy')

 def test_assert_info(self) :
     """Test the assert_info function."""
     # info_memaps should pass.
     data = npfor.open_memmap('temp.npy', mode='w+', shape=(4,3,3))
     data[:] = 5.0
     Mat = algebra.info_memmap(data)
     algebra.assert_info(Mat)
     del Mat
     os.remove('temp.npy')
     # info_arrays should pass.
     data = sp.empty((5, 6, 6))
     data[:] = 4.0
     Mat = algebra.info_array(data)
     algebra.assert_info(Mat)
     # arrays should fail.
     self.assertRaises(TypeError, algebra.assert_info, data)

    def load(self, file_path):
        """
        :param file_path: Full file path to the AequilibraEDataset to be loaded
        :return: Loads the dataset into the AequilibraEData instance
        """
        f = open(file_path)
        self.file_path = os.path.realpath(f.name)
        f.close()

        # Map in memory and load data names plus dimensions
        self.data = open_memmap(self.file_path, mode='r+')

        self.entries = self.data.shape[0]
        self.fields = [x for x in self.data.dtype.fields if x != 'index']
        self.num_fields = len(self.fields)
        self.data_types = [self.data[x].dtype.type for x in self.fields]

    def load_data_matrix(self):
        
        memmap_path = os.path.join(self.bin_dir,self.memmap_name)
        if os.path.exists(memmap_path):
            
            print 'loading in '+self.memmap_name
            
            self.raw_data_list = npf.open_memmap(memmap_path,mode='r',dtype='float32')
            #self.raw_data_list = np.load(memmap_path)

            print 'shape of loaded memmap:'
            print self.raw_data_list.shape
            
            self.loaded_warm_start = True
            return True
            
        else:
            print 'no file of name '+self.memmap_name+' to load.'
            print 'aborting memmap load'
            
            return False

    def create_data_matrix(self,save_memmap=True,nuke=True):
        
        raw_path = os.path.join(self.bin_dir,self.memmap_name)
        
        if nuke and os.path.exists(raw_path):
            os.remove(raw_path)
            
        if save_memmap:            
            # We need to determine how many nifti files there are in total to
            # determine the shape of the memmap:
            
            brainshape = []

            for subject in self.reg_subjects:
                sub_path = os.path.join(self.top_dir,subject)
                for nifti_name in self.reg_nifti_name:
                    nifti_path = os.path.join(sub_path,nifti_name)
                    if os.path.exists(nifti_path):
                        self.total_nifti_files += 1
                        if not brainshape:
                            [tempdata,tempaffine,brainshape] = self.__load_nifti(nifti_path)
                
            # Allocate the .npy memmap according to its size:
            
            memmap_shape = (self.total_nifti_files,brainshape[0],brainshape[1],brainshape[2],
                            brainshape[3])
            
            print 'Determined memmap shape:'
            print memmap_shape
            print 'Allocating the memmap...'
            
            self.raw_data_list = npf.open_memmap(raw_path,mode='w+',dtype='float32',
                                                 shape=memmap_shape)
            
            print 'Succesfully allocated memmap... memmap shape:'
            pprint(self.raw_data_list.shape)
            
        
        nifti_iter = 0
        for subject in self.reg_subjects:
            sub_path = os.path.join(self.top_dir,subject)
            print sub_path
            print subject
            print os.getcwd()
            
            for nifti_name in self.reg_nifti_name:
                nifti_path = os.path.join(sub_path,nifti_name)
                pprint(nifti_name)
                if os.path.exists(nifti_path):
                    [idata,affine,ishape] = self.__load_nifti(nifti_path)
                    pprint(ishape)
                    
                    if save_memmap:
                        print 'Appending idata to memmap at: %s' % str(nifti_iter)
                        self.raw_data_list[nifti_iter] = np.array(idata)
                        self.subject_trial_indices[nifti_iter] = []
                        nifti_iter += 1
                    
                    if self.reg_experiment_trs == False:
                        self.reg_experiment_trs = len(idata[3])
                        
                    if self.reg_total_trials == False:
                        if self.reg_trial_trs:
                            self.reg_total_trials = self.reg_experiment_trs/self.reg_trial_trs

                    if self.raw_affine == []:
                        self.raw_affine = affine
                        
                    if self.raw_data_shape == []:
                        self.raw_data_shape = ishape
                        pprint(ishape)

    ncores = 8

    savename = 'launch-%06d.npy' % (idx,)
    outname = 'reduced-%06d.pickle' % (idx,)

    msg = msgr()

    try:
        os.stat(outname)
        msg('found result file!')
    except:
        msg('no result found, proceeding to do reduction')
        msg('loading dataset %s' % savename)

        import cPickle as cp
        from numpy import *
        from numpy.linalg import svd
        from numpy.lib.format import open_memmap
        from multiprocessing import Pool

        npy = open_memmap(savename)
        npy_ = npy.reshape((-1, 32*npy.shape[1], 192))
        pool = Pool(ncores)

        svds = pool.map(reducer, range(npy_.shape[0]))

        msg('writing data')
        with open(outname, 'w') as fd:
            cp.dump(svds, fd)

from numpy.lib.format import open_memmap

n0 = open_memmap('launch-000000.npy')
n1 = open_memmap('launch-000001.npy')
n2 = open_memmap('launch-000002.npy')

n0_cond_ss = n0.reshape((-1, 32, 401, 192))[:, :, 200:].reshape((-1, 32*201, 192))
n1_cond_ss = n1.reshape((-1, 32, 401, 192))[:, :, 200:].reshape((-1, 32*201, 192))


# triple f here
figure(figsize=(15, 12))
ws = l9['dataset'].weights
ds = l9['dataset'].distances
idx = 32*10 + 21
cond = n9.reshape((-1, 32, 401, 96, 2))[idx, :, :]
ts = r_[0 : cond.shape[1]*2.5 : 1j*cond.shape[1]]
cond -= cond.reshape((-1, 192)).mean(axis=0).reshape((1, 1, 96, 2))
trial_svds = [svd(trial[:, :, 0], full_matrices=0) for trial in cond]
cond_svd = svd(cond[:, :, :, 0].reshape((-1, 96)), full_matrices=0)
for i, svdi, trial in zip(range(32), trial_svds, cond):
    subplot(335)
    x, y, z = svdi[1][:3][:, newaxis]*dot(svdi[2][:3], trial[:, :, 0].T)
    plot(x+z/3, y+z/3, 'k-', alpha=0.2)
    subplot(336)
    x, y, z = svdi[1][:3][:, newaxis]*dot(cond_svd[2][:3], trial[:, :, 0].T)
    plot(x+z/3, y+z/3, 'k-', alpha=0.3)
subplot(6,3,13)
hist(concatenate([abs(dot(svd1[2][:3], svd2[2][:3].T)).flat for i, svd1 in enumerate(trial_svds) for j, svd2 in enumerate(trial_svds) if not j==i]), 50)
xlim([0, 1.0])
subplot(3, 3, 4)

 def shape(self, item):
     """Faster version of precomputed(item).data.shape"""
     memmap = open_memmap(self.get_path(item), mode='r')
     shape = memmap.shape
     del memmap
     return shape