def create_versioned_files(inst, start=None, stop=None, freq='1D', use_doy=True,
                 root_fname=None):
    # create a bunch of files
    if start is None:
        start = pysat.datetime(2009, 1, 1)
    if stop is None:
        stop = pysat.datetime(2013, 12, 31)
    dates = pysat.utils.season_date_range(start, stop, freq=freq)

    versions = np.array([1, 2])
    revisions = np.array([0, 1])

    if root_fname is None:
        root_fname = 'pysat_testing_junk_{year:04d}_{month:02d}_{day:03d}{hour:02d}{min:02d}{sec:02d}_stuff_{version:02d}_{revision:03d}.pysat_testing_file'
    # create empty file
    for date in dates:
        for version in versions:
            for revision in revisions:
                yr, doy = pysat.utils.getyrdoy(date)
                if use_doy:
                    doy = doy
                else:
                    doy = date.day

                fname = os.path.join(inst.files.data_path, root_fname.format(year=yr,
                                                                             day=doy, month=date.month, hour=date.hour,
                                                                             min=date.minute, sec=date.second,
                                                                             version=version, revision=revision))
                with open(fname, 'w') as f:
                    pass

    def test_get_new_files_after_deleting_files_and_adding_files(self):
        # create new files and make sure that new files are captured
        start = pysat.datetime(2008, 1, 11)
        stop = pysat.datetime(2008, 1, 12)
        dates = pysat.utils.season_date_range(start, stop, freq='100min')
        # remove files, same number as will be added
        to_be_removed = len(dates)
        for the_file in os.listdir(self.testInst.files.data_path):
            if (the_file[0:13] == 'pysat_testing') & (the_file[-19:] == '.pysat_testing_file'):
                file_path = os.path.join(self.testInst.files.data_path, the_file)
                if os.path.isfile(file_path) & (to_be_removed > 0):
                    to_be_removed -= 1
                    # Remove all versions of the file
                    # otherwise, previous versions will look like new files
                    pattern = '_'.join(file_path.split('_')[0:7])+'*.pysat_testing_file'
                    map(os.unlink, glob.glob(pattern))
                    #os.unlink(file_path)
        # add new files
        create_versioned_files(self.testInst, start, stop, freq='100min',
                     use_doy=False,
                     root_fname=self.root_fname)
        # get new files
        new_files = self.testInst.files.get_new()

        assert (np.all(new_files.index == dates))

    def test_refresh_on_unchanged_files(self):

        start = pysat.datetime(2007, 12, 31)
        stop = pysat.datetime(2008, 1, 10)
        dates = pysat.utils.season_date_range(start, stop, freq='100min')
        self.testInst.files.refresh()
        assert (np.all(self.testInst.files.files.index == dates))

 def test_set_bounds_by_date_season(self):
     start = [pysat.datetime(2009,1,1), pysat.datetime(2009,2,1)]
     stop = [pysat.datetime(2009,1,15), pysat.datetime(2009,2,15)]
     self.testInst.bounds = (start, stop)
     out = pds.date_range(start[0], stop[0]).tolist()
     out.extend(pds.date_range(start[1], stop[1]).tolist())
     assert np.all(self.testInst._iter_list == out)

 def test_basic_orbit_mean(self):
     orbit_info = {'kind':'local time', 'index':'mlt'}
     self.testInst = pysat.Instrument('pysat','testing', clean_level='clean', orbit_info=orbit_info)      
     self.testInst.bounds = (pysat.datetime(2009,1,1), pysat.datetime(2009,1,2))
     ans = pysat.ssnl.avg.mean_by_orbit(self.testInst, 'mlt')
     # note last orbit is incomplete thus not expected to satisfy relation
     assert np.allclose(ans[:-1], np.ones(len(ans)-1)*12., 1.E-2)

    def test_heterogenous_constellation_average(self):
        for inst in self.testC:
            inst.bounds = (pysat.datetime(2008,1,1), pysat.datetime(2008,2,1))
        results = pysat.ssnl.avg.median2D(self.testC, [0., 360., 24.], 'longitude',
                                          [0., 24, 24], 'mlt', ['dummy1', 'dummy2', 'dummy3'])
        dummy_val = results['dummy1']['median']
        dummy_dev = results['dummy1']['avg_abs_dev']

        dummy2_val = results['dummy2']['median']
        dummy2_dev = results['dummy2']['avg_abs_dev']

        dummy3_val = results['dummy3']['median']
        dummy3_dev = results['dummy3']['avg_abs_dev']
        
        dummy_x = results['dummy1']['bin_x']
        dummy_y = results['dummy1']['bin_y']
        
        # iterate over all y rows, value should be equal to integer value of mlt
        # no variation in the median, all values should be the same
        check = []
        for i, y in enumerate(dummy_y[:-1]):
            check.append(np.all(dummy_val[i, :] == y.astype(int)))
            check.append(np.all(dummy_dev[i, :] == 0))

        for i, x in enumerate(dummy_x[:-1]):
            check.append(np.all(dummy2_val[:, i] == x/15.) )
            check.append(np.all(dummy2_dev[:, i] == 0))

        for i, x in enumerate(dummy_x[:-1]):
            check.append(np.all(dummy3_val[:, i] == x/15.*1000. + dummy_y[:-1]) )
            check.append(np.all(dummy3_dev[:, i] == 0))

        assert np.all(check)

    def setup(self):
        """Runs before every method to create a clean testing setup."""
        # store current pysat directory
        self.data_path = pysat.data_dir
        # create temporary directory
        dir_name = tempfile.gettempdir()
        pysat.utils.set_data_dir(dir_name, store=False)
        # create testing directory
        create_dir(temporary_file_list=self.temporary_file_list)

        # create a test instrument, make sure it is getting files from filesystem
        re_load(pysat.instruments.pysat_testing)
        # self.stored_files_fcn = pysat.instruments.pysat_testing.list_files
        pysat.instruments.pysat_testing.list_files = list_versioned_files
        # create a bunch of files by year and doy
        self.testInst = pysat.Instrument(inst_module=pysat.instruments.pysat_testing,
                                         clean_level='clean',
                                         temporary_file_list=self.temporary_file_list)

        self.root_fname = 'pysat_testing_junk_{year:04d}_{month:02d}_{day:03d}{hour:02d}{min:02d}{sec:02d}_stuff_{version:02d}_{revision:03d}.pysat_testing_file'
        start = pysat.datetime(2007, 12, 31)
        stop = pysat.datetime(2008, 1, 10)
        create_versioned_files(self.testInst, start, stop, freq='100min',
                     use_doy=False,
                     root_fname=self.root_fname)

        self.testInst = pysat.Instrument(inst_module=pysat.instruments.pysat_testing,
                                         clean_level='clean', update_files=True,
                                         temporary_file_list=self.temporary_file_list)

    def test_files_non_standard_pysat_directory(self):
        # create new files and make sure that new files are captured
        start = pysat.datetime(2008, 1, 11)
        stop = pysat.datetime(2008, 1, 15)
        dates = pysat.utils.season_date_range(start, stop, freq='100min')
        pysat.instruments.pysat_testing.list_files = list_versioned_files
        self.testInst = pysat.Instrument(inst_module=pysat.instruments.pysat_testing,
                                         clean_level='clean',
                                         sat_id='hello',
                                         directory_format='pysat_testing_{tag}_{sat_id}',
                                         update_files=True,
                                         temporary_file_list=self.temporary_file_list)
        # add new files
        create_dir(self.testInst)
        remove_files(self.testInst)
        create_versioned_files(self.testInst, start, stop, freq='100min',
                     use_doy=False,
                     root_fname=self.root_fname)

        self.testInst = pysat.Instrument(inst_module=pysat.instruments.pysat_testing,
                                         clean_level='clean',
                                         sat_id='hello',
                                         directory_format='pysat_testing_{tag}_{sat_id}',
                                         update_files=True,
                                         temporary_file_list=self.temporary_file_list)

        # get new files
        new_files = self.testInst.files.get_new()
        assert (np.all(self.testInst.files.files.index == dates) &
                np.all(new_files.index == dates))

 def test_iterate_over_default_bounds(self):
     start = pysat.datetime(2008,1,1)
     stop = pysat.datetime(2010,12,31)
     self.testInst.bounds = (start, stop)
     dates = []
     for inst in self.testInst:
         dates.append(inst.date)            
     out = pds.date_range(start, stop).tolist()
     assert np.all(dates == out)

 def test_iterate_over_bounds_set_by_date_season(self):
     start = [pysat.datetime(2009,1,1), pysat.datetime(2009,2,1)]
     stop = [pysat.datetime(2009,1,15), pysat.datetime(2009,2,15)]
     self.testInst.bounds = (start, stop)
     dates = []
     for inst in self.testInst:
         dates.append(inst.date)            
     out = pds.date_range(start[0], stop[0]).tolist()
     out.extend(pds.date_range(start[1], stop[1]).tolist())
     assert np.all(dates == out)

    def test_get_new_files_after_adding_files(self):
        # create new files and make sure that new files are captured
        start = pysat.datetime(2008, 1, 11)
        stop = pysat.datetime(2008, 1, 12)

        create_versioned_files(self.testInst, start, stop, freq='100min',
                     use_doy=False,
                     root_fname=self.root_fname)
        dates = pysat.utils.season_date_range(start, stop, freq='100min')
        new_files = self.testInst.files.get_new()
        assert (np.all(new_files.index == dates))

 def test_iterate_over_bounds_set_by_fname(self):
     start = '2009-01-01.nofile'
     stop = '2009-01-15.nofile'
     start_d = pysat.datetime(2009,1,1)
     stop_d = pysat.datetime(2009,1,15)
     self.testInst.bounds = (start, stop)
     dates = []
     for inst in self.testInst:
         dates.append(inst.date)            
     out = pds.date_range(start_d, stop_d).tolist()
     assert np.all(dates == out)

 def test_iterate_over_bounds_set_by_fname_season(self):
     start = ['2009-01-01.nofile', '2009-02-01.nofile']
     stop = ['2009-01-15.nofile', '2009-02-15.nofile']
     start_d = [pysat.datetime(2009,1,1), pysat.datetime(2009,2,1)]
     stop_d = [pysat.datetime(2009,1,15), pysat.datetime(2009,2,15)]
     self.testInst.bounds = (start, stop)
     dates = []
     for inst in self.testInst:
         dates.append(inst.date)            
     out = pds.date_range(start_d[0], stop_d[0]).tolist()
     out.extend(pds.date_range(start_d[1], stop_d[1]).tolist())
     assert np.all(dates == out)

    def test_refresh(self):
        # create new files and make sure that new files are captured
        start = pysat.datetime(2008,1,10)
        stop = pysat.datetime(2008,1,12)

        create_files(self.testInst, start, stop, freq='100min',  
                     use_doy=False, 
                     root_fname = self.root_fname)
        start = pysat.datetime(2007,12,31)
        dates = pysat.utils.season_date_range(start, stop, freq='100min')
        self.testInst.files.refresh()
        assert (np.all(self.testInst.files.files.index == dates))

 def test_iterate_over_bounds_set_by_fname_via_prev(self):
     start = '2009-01-01.nofile'
     stop = '2009-01-15.nofile'
     start_d = pysat.datetime(2009,1,1)
     stop_d = pysat.datetime(2009,1,15)
     self.testInst.bounds = (start, stop)
     dates = []
     self.testInst.prev()
     dates.append(self.testInst.date) 
     while self.testInst.date > start_d:
         self.testInst.prev()
         dates.append(self.testInst.date)            
     out = pds.date_range(start_d, stop_d).tolist()
     assert np.all(dates == out[::-1])

    def setup(self):
        '''Runs before every method to create a clean testing setup.'''
        info = {'index':'mlt'}
        self.testInst = pysat.Instrument('pysat','testing', '86400', 
                                        clean_level='clean',
                                        orbit_info=info)
        times = [ [pysat.datetime(2008,12,31,4), pysat.datetime(2008,12,31,5,37)],              
                [pysat.datetime(2009,1,1), pysat.datetime(2009,1,1,1,37)]
                ]
        for seconds in np.arange(38):
            day = pysat.datetime(2009,1,2)+pds.DateOffset(days=int(seconds))
            times.append([day, day+pds.DateOffset(hours=1, minutes=37, seconds=int(seconds))-pds.DateOffset(seconds=20)])

        self.testInst.custom.add(filter_data2, 'modify', times=times)

 def test_year_doy_files_direct_call_to_from_os(self):
     # create a bunch of files by year and doy
     start = pysat.datetime(2008,1,1)
     stop = pysat.datetime(2009,12,31)
     create_files(self.testInst, start, stop, freq='1D')
     # use from_os function to get pandas Series of files and dates
     files = pysat.Files.from_os(data_path=self.testInst.files.data_path,
         format_str='pysat_testing_junk_{year:04d}_gold_{day:03d}_stuff.pysat_testing_file')
     # check overall length
     check1 = len(files) == (365+366)  
     # check specific dates
     check2 = pds.to_datetime(files.index[0]) == pysat.datetime(2008,1,1)
     check3 = pds.to_datetime(files.index[365]) == pysat.datetime(2008,12,31)
     check4 = pds.to_datetime(files.index[-1]) == pysat.datetime(2009,12,31)
     assert(check1 & check2 & check3 & check4)

 def test_next_fid_load_default(self):
     """Test next day is being loaded (checking object date)."""
     self.testInst.load(fid=0)
     self.testInst.next()
     test_date = self.testInst.data.index[0]
     test_date = pysat.datetime(test_date.year, test_date.month, test_date.day)
     assert (test_date == pds.datetime(2008,1,2)) & (test_date == self.testInst.date)

 def test_next_filename_load_default(self):
     """Test next day is being loaded (checking object date)."""
     self.testInst.load(fname='2010-12-30.nofile')
     self.testInst.next()
     test_date = self.testInst.data.index[0]
     test_date = pysat.datetime(test_date.year, test_date.month, test_date.day)
     assert (test_date == pds.datetime(2010,12,31)) & (test_date == self.testInst.date)

    def test_instrument_has_files(self):
        import pysat.instruments.pysat_testing

        root_fname='pysat_testing_junk_{year:04d}_gold_{day:03d}_stuff_{month:02d}_{hour:02d}_{min:02d}_{sec:02d}.pysat_testing_file'
        # create a bunch of files by year and doy
        start = pysat.datetime(2007,12,31)
        stop = pysat.datetime(2008,1,10)
        create_files(self.testInst, start, stop, freq='100min',  
                     use_doy=False, 
                     root_fname = root_fname)
        # create the same range of dates
        dates = pysat.utils.season_date_range(start, stop, freq='100min')
        pysat.instruments.pysat_testing.list_files = list_files
        inst = pysat.Instrument(platform='pysat', name='testing', update_files=True)
        re_load(pysat.instruments.pysat_testing)
        assert (np.all(inst.files.files.index == dates))