def test_store_and_load_large_dictionary(self):
        traj = Trajectory(name='Testlargedict', filename=make_temp_dir('large_dict.hdf5'))

        large_dict = {}

        for irun in range(1025):
            large_dict['item_%d' % irun] = irun

        large_dict2 = {}

        for irun in range(33):
            large_dict2['item_%d' % irun] = irun

        traj.f_add_result('large_dict', large_dict, comment='Huge_dict!')
        traj.f_add_result('large_dict2', large_dict2, comment='Not so large dict!')

        traj.f_store()

        traj_name = traj.v_name

        traj2 = Trajectory(filename=make_temp_dir('large_dict.hdf5'))

        traj2.f_load(name=traj_name, load_data=2)

        self.compare_trajectories(traj, traj2)

    def make_environment(self, idx, filename):



        #self.filename = '../../experiments/tests/HDF5/test.hdf5'
        self.logfolder = make_temp_dir(os.path.join('experiments',
                                                     'tests',
                                                     'Log'))
        self.cnt_folder = make_temp_dir(os.path.join('experiments','tests','cnt'))
        trajname = 'Test%d' % idx + '_' + make_trajectory_name(self)

        env = Environment(trajectory=trajname,
                          filename=filename,
                          file_title=trajname,
                          log_stdout=False,

                          log_config=get_log_config(),
                          continuable=True,
                          continue_folder=self.cnt_folder,
                          delete_continue=False,
                          large_overview_tables=True)


        self.envs.append(env)
        self.trajs.append( env.v_trajectory)

    def make_environment_mp(self, idx, filename):
        #self.filename = '../../experiments/tests/HDF5/test.hdf5'
        self.logfolder = make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'Log'))
        self.cnt_folder = make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'cnt'))
        trajname = 'Test%d' % idx + '_' + make_trajectory_name(self)

        env = Environment(trajectory=trajname,
                          dynamic_imports=[CustomParameter],
                          filename=filename,
                          file_title=trajname,
                          log_stdout=False,
                          purge_duplicate_comments=False,
                          log_config=get_log_config(),
                          continuable=True,
                          continue_folder=self.cnt_folder,
                          delete_continue=False,
                          multiproc=True,
                          use_pool=True,
                          ncores=4)

        self.envs.append(env)
        self.trajs.append( env.v_trajectory)

    def test_store_items_and_groups(self):

        traj = Trajectory(name='testtraj', filename=make_temp_dir('teststoreitems.hdf5'))

        traj.f_store()

        traj.f_add_parameter('group1.test',42, comment= 'TooLong' * pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH)

        traj.f_add_result('testres', 42)

        traj.group1.f_set_annotations(Test=44)

        traj.f_store_items(['test','testres','group1'])


        traj2 = Trajectory(name=traj.v_name, add_time=False,
                           filename=make_temp_dir('teststoreitems.hdf5'))

        traj2.f_load(load_parameters=2, load_results=2)

        traj.f_add_result('Im.stored.along.a.path', 43)
        traj.Im.stored.along.v_annotations['wtf'] =4444
        traj.res.f_store_child('Im.stored.along.a.path')


        traj2.res.f_load_child('Im.stored.along.a.path', load_data=2)

        self.compare_trajectories(traj,traj2)

    def test_version_mismatch(self):
        traj = Trajectory(name='TestVERSION', filename=make_temp_dir('testversionmismatch.hdf5'),
                          add_time=True)

        traj.f_add_parameter('group1.test',42)

        traj.f_add_result('testres', 42)

        traj.group1.f_set_annotations(Test=44)

        traj._version='0.1a.1'

        traj.f_store()

        traj2 = Trajectory(name=traj.v_name, add_time=False,
                           filename=make_temp_dir('testversionmismatch.hdf5'))

        with self.assertRaises(pex.VersionMismatchError):
            traj2.f_load(load_parameters=2, load_results=2)

        traj2.f_load(load_parameters=2, load_results=2, force=True)

        self.compare_trajectories(traj,traj2)

        get_root_logger().info('Mismatch testing done!')

    def test_logging_stdout(self):
        filename = 'teststdoutlog.hdf5'
        filename = make_temp_dir(filename)
        folder = make_temp_dir('logs')
        env = Environment(trajectory=make_trajectory_name(self),
                          filename=filename, log_config=get_log_config(),
                          # log_levels=logging.CRITICAL, # needed for the test
                          log_stdout=('STDOUT', 50), #log_folder=folder
                          )

        env.f_run(log_error)
        traj = env.v_traj
        path = get_log_path(traj)

        mainstr = 'sTdOuTLoGGinG'
        print(mainstr)
        env.f_disable_logging()

        mainfilename = os.path.join(path, 'LOG.txt')
        with open(mainfilename, mode='r') as mainf:
            full_text = mainf.read()

        self.assertTrue(mainstr in full_text)
        self.assertTrue('4444444' not in full_text)
        self.assertTrue('DEBUG' not in full_text)

    def test_continueing_mp_custom(self):
        self.filenames = [make_temp_dir('test_continueing_mp_custom.hdf5'),
                          make_temp_dir('test_continueing_mp_custom2.hdf5')]

        self.envs=[]
        self.trajs = []

        for irun,filename in enumerate(self.filenames):
            if isinstance(filename,int):
                filename = self.filenames[filename]

            self.make_environment(irun, filename, continuable=irun == 1)

        self.param_dict={'x':1.0, 'y':2.0}

        for irun in range(len(self.filenames)):
            self.trajs[irun].f_add_parameter(CustomParameter,'x', 1.0)
            self.trajs[irun].f_add_parameter(CustomParameter, 'y', 1.0)


        self.explore_mp(self.trajs[0])
        self.explore_mp(self.trajs[1])

        arg=33
        for irun in range(len(self.filenames)):

            self.envs[irun].f_run(Multiply(), arg)

        traj_name = self.trajs[1].v_name
        continue_folder = os.path.join(self.cnt_folder, self.trajs[1].v_name)

        self.envs.pop()
        self.assertEqual(len(self.envs), 1)
        self.make_environment(1, self.filenames[1], continuable=True, add_time=False,
                              trajectory=traj_name)
        self.trajs[1] = self.envs[1].v_traj
        self.trajs[1].f_load(load_data=pypetconstants.LOAD_NOTHING)
        self._remove_nresults(self.trajs[1], 3, continue_folder)

        results = self.envs[1].f_continue(trajectory_name = traj_name)
        results = sorted(results, key = lambda x: x[0])

        for irun in range(len(self.filenames)):
            self.trajs[irun].f_load(load_parameters=pypetconstants.OVERWRITE_DATA,
                                    load_derived_parameters=pypetconstants.OVERWRITE_DATA,
                                    load_results=pypetconstants.OVERWRITE_DATA,
                                    load_other_data=pypetconstants.OVERWRITE_DATA)

        self.compare_trajectories(self.trajs[1],self.trajs[0])

        for run_name in self.trajs[0].f_iter_runs():
            z = (self.trajs[0].v_idx, self.trajs[0].crun.z)
            self.assertTrue(z in results, '%s not in %s' % (z, results))

        self.assertTrue(len(self.trajs[-1])== len(results))

    def setUp(self):
        self.set_mode()
        self.logfolder = make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'Log'))

        random.seed()
        self.trajname = make_trajectory_name(self)
        self.filename = make_temp_dir(os.path.join('experiments',
                                                    'tests',
                                                    'HDF5',
                                                    'test%s.hdf5' % self.trajname))

        env = Environment(trajectory=self.trajname, filename=self.filename,
                          file_title=self.trajname,
                          log_stdout=self.log_stdout,
                          log_config=get_log_config(),
                          results_per_run=5,
                          wildcard_functions=self.wildcard_functions,
                          derived_parameters_per_run=5,
                          multiproc=self.multiproc,
                          ncores=self.ncores,
                          wrap_mode=self.mode,
                          use_pool=self.use_pool,
                          gc_interval=self.gc_interval,
                          freeze_input=self.freeze_input,
                          fletcher32=self.fletcher32,
                          complevel=self.complevel,
                          complib=self.complib,
                          shuffle=self.shuffle,
                          pandas_append=self.pandas_append,
                          pandas_format=self.pandas_format,
                          encoding=self.encoding,
                          niceness=self.niceness,
                          use_scoop=self.use_scoop,
                          port=self.port,
                          add_time=self.add_time,
                          timeout=self.timeout,
                          graceful_exit=self.graceful_exit)

        traj = env.v_trajectory

        traj.v_standard_parameter=Parameter

        ## Create some parameters
        self.param_dict={}
        create_param_dict(self.param_dict)
        ### Add some parameter:
        add_params(traj,self.param_dict)

        #remember the trajectory and the environment
        self.traj = traj
        self.env = env

    def setUp(self):

        self.multiproc = True
        self.mode = 'LOCK'

        self.trajname = make_trajectory_name(self)

        self.filename = make_temp_dir(os.path.join('experiments',
                                                    'tests',
                                                    'HDF5',
                                                    '%s.hdf5' % self.trajname))
        self.logfolder = make_temp_dir(os.path.join('experiments', 'tests', 'Log'))

        random.seed()

        cap_dicts = (dict(cpu_cap=0.000001), # Ensure that these are triggered
                      dict(memory_cap=(0.000001, 150.0)),
                      dict(swap_cap=0.000001,))

        cap_dict = cap_dicts[CapTest.cap_count]
        env = Environment(trajectory=self.trajname,filename=self.filename,
                          file_title=self.trajname, log_folder=self.logfolder,
                          logger_names=('pypet', 'test'), log_levels='ERROR',
                          log_stdout=False,
                          results_per_run=5,
                          derived_parameters_per_run=5,
                          multiproc=True,
                          ncores=4,
                          use_pool=False,
                          niceness = check_nice(11),
                          **cap_dict)

        logging.getLogger('test').error('Using Cap: %s and file: %s' % (str(cap_dict),
                                                                        str(self.filename)))
        # Loop through all possible cap configurations
        # and test one at a time
        CapTest.cap_count += 1
        CapTest.cap_count = CapTest.cap_count % len(cap_dicts)

        traj = env.v_trajectory

        ## Create some parameters
        self.param_dict={}
        create_param_dict(self.param_dict)
        ### Add some parameter:
        add_params(traj,self.param_dict)

        #remember the trajectory and the environment
        self.traj = traj
        self.env = env

 def test_merge_basic_with_separate_files_only_adding_more_trials(self):
     self.filenames = [make_temp_dir(os.path.join('experiments',
                                      'tests',
                                      'HDF5',
                                      'merge2.hdf5')),
                       make_temp_dir(os.path.join('experiments',
                                      'tests',
                                      'HDF5',
                                      'merge3.hdf5')),
                       make_temp_dir(os.path.join('experiments',
                                      'tests',
                                      'HDF5',
                                      'merge4.hdf5'))]
     self.merge_basic_only_adding_more_trials(True)

 def test_basic_within_separate_file_and_skipping_duplicates_which_leads_to_one_reamianing(self):
     self.filenames = [make_temp_dir(os.path.join('experiments',
                                      'tests',
                                      'HDF5',
                                      'merge2.hdf5')),
                       make_temp_dir(os.path.join('experiments',
                                      'tests',
                                      'HDF5',
                                      'merge3.hdf5')),
                       make_temp_dir(os.path.join('experiments',
                                      'tests',
                                      'HDF5',
                                      'merge4.hdf5'))]
     self. basic_and_skipping_duplicates_which_leads_to_one_remaining()

    def setUp(self):
        self.set_mode()

        logging.basicConfig(level=logging.ERROR)


        self.logfolder = make_temp_dir(os.path.join('experiments',
                                                      'tests',
                                                      'Log'))

        random.seed()
        self.trajname = make_trajectory_name(self)
        self.filename = make_temp_dir(os.path.join('experiments',
                                                    'tests',
                                                    'HDF5',
                                                    'test%s.hdf5' % self.trajname))

        env = Environment(trajectory=self.trajname, filename=self.filename,
                          file_title=self.trajname,
                          log_stdout=self.log_stdout,

                          log_config=get_log_config(),
                          results_per_run=5,
                          derived_parameters_per_run=5,
                          multiproc=self.multiproc,
                          ncores=self.ncores,
                          wrap_mode=self.mode,
                          use_pool=self.use_pool,
                          fletcher32=self.fletcher32,
                          complevel=self.complevel,
                          complib=self.complib,
                          shuffle=self.shuffle,
                          pandas_append=self.pandas_append,
                          pandas_format=self.pandas_format,
                          encoding=self.encoding)

        traj = env.v_trajectory

        traj.v_standard_parameter=Parameter

        ## Create some parameters
        create_link_params(traj)
        ### Add some parameter:
        explore_params(traj)

        #remember the trajectory and the environment
        self.traj = traj
        self.env = env

    def test_errors(self):
        filename = make_temp_dir("hdf5errors.hdf5")
        traj = Trajectory(name=make_trajectory_name(self), filename=filename)
        trajname = traj.v_name

        npearray = np.ones((2, 10, 3), dtype=np.float)
        thevlarray = np.array([compat.tobytes("j"), 22.2, compat.tobytes("gutter")])

        with self.assertRaises(TypeError):
            traj.f_add_result(SharedResult, "arrays.vlarray", SharedVLArray()).create_shared_data(obj=thevlarray)
        traj.f_store()
        traj.arrays.vlarray.create_shared_data(obj=thevlarray)
        traj.f_add_result(SharedResult, "arrays.array", SharedArray()).create_shared_data(data=npearray)
        traj.arrays.f_add_result(SharedResult, "super.carray", SharedCArray(), comment="carray").create_shared_data(
            shape=(10, 10), atom=pt.atom.FloatAtom()
        )
        traj.arrays.f_add_result(SharedResult, "earray", SharedEArray()).create_shared_data("earray", obj=npearray)

        traj.f_store()

        with self.assertRaises(TypeError):
            traj.arrays.array.iter_rows()

        with StorageContextManager(traj) as cm:
            with self.assertRaises(RuntimeError):
                with StorageContextManager(traj) as cm2:
                    pass
            self.assertTrue(traj.v_storage_service.is_open)
            with self.assertRaises(RuntimeError):
                StorageContextManager(traj).f_open_store()

        self.assertFalse(traj.v_storage_service.is_open)

    def test_hdf5_store_load_parameter(self):
        traj_name = make_trajectory_name(self)
        file_name = make_temp_dir(os.path.join('brian2', 'tests', 'hdf5', 'test_%s.hdf5' % traj_name))
        env = Environment(trajectory=traj_name, filename=file_name, log_config=get_log_config(),
                            dynamic_imports=[Brian2Parameter], add_time=False, storage_service=HDF5StorageService)
        traj = env.v_trajectory
        traj.v_standard_parameter = Brian2Parameter
        traj.f_add_parameter('brian2.single.millivolts', 10*mvolt, comment='single value')

        #traj.f_add_parameter('brian2.array.millivolts', [11, 12]*mvolt, comment='array')
        #traj.f_add_parameter('mV1', 42.0*mV)
        #traj.f_add_parameter('ampere1', 1*mA)
        #traj.f_add_parameter('integer', 16)
        #traj.f_add_parameter('kHz05', 0.5*kHz)
        #traj.f_add_parameter('nested_array', np.array([[6.,7.,8.],[9.,10.,11.]]) * ms)
        #traj.f_add_parameter('b2a', np.array([1., 2.]) * mV)

        # We also need to check if explorations work with hdf5 store!
        #explore_dict = {'ampere1': [1*mA, 2*mA, 3*mA],
        #                'integer': [42,43,44],
        #                'b2a': [np.array([1., 2.]) * mV, np.array([1., 4.]) * mV,
        #                       np.array([1., 2.]) * mV]}
        #traj.f_explore(explore_dict)

        traj.f_store()

        traj2 = load_trajectory(filename=file_name, name=traj_name, dynamic_imports=[Brian2Parameter],
                                load_data=2)
        self.compare_trajectories(traj, traj2)

    def test_hdf5_store_load_monitorresult(self):
        traj_name = make_trajectory_name(self)
        file_name = make_temp_dir(os.path.join('brian2', 'tests', 'hdf5', 'test_%s.hdf5' % traj_name))
        env = Environment(trajectory=traj_name, filename=file_name, log_config=get_log_config(),
                            dynamic_imports=[Brian2MonitorResult], add_time=False, storage_service=HDF5StorageService)
        traj = env.v_trajectory
        traj.v_standard_result = Brian2MonitorResult
        traj.f_add_result('brian2.single.millivolts_single_a', 10*mvolt, comment='single value a')

        traj.f_add_result('brian2.single.millivolts_single_c', 11*mvolt, comment='single value b')

        traj.f_add_result('brian2.array.millivolts_array_a', [11, 12]*mvolt, comment='array')
        traj.f_add_result('mV1', 42.0*mV)
        # results can hold much more than a single data item:
        traj.f_add_result('ampere1', 1*mA, 44, test=300*mV, test2=[1,2,3],
                          test3=np.array([1,2,3])*mA, comment='Result keeping track of many things')
        traj.f_add_result('integer', 16)
        traj.f_add_result('kHz05', 0.5*kHz)
        traj.f_add_result('nested_array', np.array([[6.,7.,8.],[9.,10.,11.]]) * ms)
        traj.f_add_result('b2a', np.array([1., 2.]) * mV)


        traj.f_store()

        traj2 = load_trajectory(filename=file_name, name=traj_name, dynamic_imports=[Brian2MonitorResult], load_data=2)

        #traj._logger.error('traj :'+str(traj))
        #traj._logger.error('traj2:'+str(traj2))
        self.compare_trajectories(traj, traj2)

    def test_removal_of_error_parameter(self):

        filename = make_temp_dir('remove_errored.hdf5')
        traj = Trajectory(name='traj', add_time=True, filename=filename)
        traj.f_add_result('iii', 42)
        traj.f_add_result(FakeResult, 'j.j.josie', 43)

        file = traj.v_storage_service.filename
        traj.f_store(only_init=True)
        with self.assertRaises(RuntimeError):
            traj.f_store()

        with ptcompat.open_file(file, mode='r') as fh:
            jj = ptcompat.get_node(fh, where='/%s/results/j/j' % traj.v_name)
            self.assertTrue('josie' not in jj)

        traj.j.j.f_remove_child('josie')
        traj.j.j.f_add_result(FakeResult2, 'josie2', 444)

        traj.f_store()
        with self.assertRaises(pex.NoSuchServiceError):
            traj.f_store_child('results', recursive=True)

        with ptcompat.open_file(file, mode='r') as fh:
            jj = ptcompat.get_node(fh, where='/%s/results/j/j' % traj.v_name)
            self.assertTrue('josie2' in jj)
            josie2 = ptcompat.get_child(jj, 'josie2')
            self.assertTrue('hey' in josie2)
            self.assertTrue('fail' not in josie2)

    def setUp(self):
        env = Environment(trajectory='Test_'+repr(time.time()).replace('.','_'),
                          filename=make_temp_dir(os.path.join(
                              'experiments',
                              'tests',
                              'briantests',
                              'HDF5',
                               'briantest.hdf5')),
                          file_title='test',
                          log_config=get_log_config(),
                          dynamic_imports=['pypet.brian.parameter.BrianParameter',
                                                        BrianMonitorResult],
                          multiproc=False)

        traj = env.v_trajectory

        #env._set_standard_storage()
        #env._hdf5_queue_writer._hdf5storageservice = LazyStorageService()
        traj = env.v_trajectory
        #traj.set_storage_service(LazyStorageService())

        add_params(traj)
        #traj.mode='Parallel'


        traj.f_explore(cartesian_product({traj.f_get('N').v_full_name:[50,60],
                               traj.f_get('tauw').v_full_name:[30*ms,40*ms]}))

        self.traj = traj

        self.env = env
        self.traj = traj

    def test_overwrite_stuff(self):
        traj = Trajectory(name='TestOverwrite', filename=make_temp_dir('testowrite.hdf5'))

        res = traj.f_add_result('mytest.test', a='b', c='d')

        traj.f_store()

        res['a'] = np.array([1,2,3])
        res['c'] = 123445

        traj.f_store_item(res, overwrite='a', complevel=4)

        # Should emit a warning
        traj.f_store_item(res, overwrite=['a', 'b'])

        traj.f_load(load_results=3)

        res = traj.test

        self.assertTrue((res['a']==np.array([1,2,3])).all())
        self.assertTrue(res['c']=='d')

        res['c'] = 123445

        traj.f_store_item(res, store_data=3)
        res.f_empty()

        traj.f_load(load_results=3)

        self.assertTrue(traj.test['c']==123445)

    def test_maximum_overview_size(self):

        filename = make_temp_dir('maxisze.hdf5')

        env = Environment(trajectory='Testmigrate', filename=filename,

                          log_config=get_log_config())

        traj = env.v_trajectory
        for irun in range(pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH):
            traj.f_add_parameter('f%d.x' % irun, 5)

        traj.f_store()


        store = ptcompat.open_file(filename, mode='r+')
        table = ptcompat.get_child(store.root,traj.v_name).overview.parameters_overview
        self.assertEquals(table.nrows, pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH)
        store.close()

        for irun in range(pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH,
                  2*pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH):
            traj.f_add_parameter('f%d.x' % irun, 5)

        traj.f_store()

        store = ptcompat.open_file(filename, mode='r+')
        table = ptcompat.get_child(store.root,traj.v_name).overview.parameters_overview
        self.assertEquals(table.nrows, pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH)
        store.close()

        env.f_disable_logging()

 def test_errors(self):
     tmp = make_temp_dir('cont')
     if dill is not None:
         env1 = Environment(continuable=True, continue_folder=tmp,
                        log_config=None, filename=self.filename)
         with self.assertRaises(ValueError):
             env1.f_run_map(multiply_args, [1], [2], [3])
     with self.assertRaises(ValueError):
         Environment(multiproc=True, use_pool=False, freeze_input=True,
                        filename=self.filename, log_config=None)
     env3 = Environment(log_config=None, filename=self.filename)
     with self.assertRaises(ValueError):
         env3.f_run_map(multiply_args)
     with self.assertRaises(ValueError):
         Environment(use_scoop=True, immediate_postproc=True)
     with self.assertRaises(ValueError):
         Environment(use_pool=True, immediate_postproc=True)
     with self.assertRaises(ValueError):
         Environment(continuable=True, wrap_mode='QUEUE', continue_folder=tmp)
     with self.assertRaises(ValueError):
         Environment(use_scoop=True, wrap_mode='QUEUE')
     with self.assertRaises(ValueError):
         Environment(automatic_storing=False,
                     continuable=True, continue_folder=tmp)
     with self.assertRaises(ValueError):
         Environment(port='www.nosi.de', wrap_mode='LOCK')