def assertMetadataRecorded(self, expected):
        if self.comm.rank != 0:
            return

        db = SqliteDict(self.filename, self.tablename_metadata)
        _assertMetadataRecorded(self, db, expected)
        db.close()

    def __init__(self, bucket_name, storage_path=None):
        ''' Bucker init

        - if the bucket exists, meta parameter will be ignored

        '''
        if bucket_name and isinstance(bucket_name, (str, unicode)) and re.match(r"^[a-z0-9\.\-_]+$", bucket_name, re.I):
            self._name = bucket_name.strip()
        else:
            raise falcon.HTTPInvalidParam(
                "The parameter shall contain only alpha-numeric characters, value: '%s'" % bucket_name, 
                param_name='name'
            )

        self._bucket_path = None
        if storage_path and os.path.exists(storage_path):
            self._bucket_path = os.path.join(storage_path, self._name)
        else:
            raise falcon.HTTPInternalServerError(
                title='IncorrectStoragePath',
                description='The storage path is incorrect, "%s"' % storage_path
            )

        if self._bucket_path and os.path.exists(self._bucket_path):
            self._meta = SqliteDict(os.path.join(self._bucket_path,'metadata.sqlite'), 'bucket', autocommit=True)
        else:
            self._meta = SqliteDict(':memory:', 'bucket', autocommit=True)

def adjust_evernote_font():
    """
    Call for Evernote
    """
    note_info = SqliteDict(conf.db.db_file, autocommit=True)

    notes_in_evernote = list()
    for note in get_notes(get_notebooks()):
        guid = note.guid
        notes_in_evernote.append(guid)
        if guid not in note_info.keys() \
                or note_info[guid][FONT_SIZE] != conf.font_size \
                or note_info[guid][LINE_HEIGHT] != conf.line_height:
            adjust_note(note)
            note_info[guid] = {FONT_SIZE: conf.font_size,
                               LINE_HEIGHT: conf.line_height}

    guids_to_forget = [guid for guid in note_info.keys()
                       if guid not in notes_in_evernote]

    for guid in guids_to_forget:
        logging.debug("Delete guid from DB: {}".format(guid))
        del note_info[guid]

    note_info.close()

    def assertIterationDataRecorded(self, expected, tolerance, root):
        if self.comm.rank != 0:
            return

        db = SqliteDict(self.filename, self.tablename_iterations)
        _assertIterationDataRecorded(self, db, expected, tolerance)
        db.close()

def _persist_v0(file_path, zg):
    print 'Creating db...'
    persisted = SqliteDict(file_path, autocommit=False)
    print 'Updating data...'
    persisted.update(zg.country_postal_codes)
    print 'Committing data...'
    persisted.commit()

def _import_sql_data(data_dir):
    file_path = os.path.join(data_dir, DATA_FILE)

    # Find out what format we have
    with sqlite3.connect(file_path) as conn:
        try:
            conn.execute('select count(*) from zipgun_info')
            zipgun_info = SqliteDict(file_path, tablename='zipgun_info')
            version = zipgun_info.get('version', 0)
        except sqlite3.OperationalError:
            version = 0

    if version == 0:
        country_postal_codes = SqliteDict(file_path)
    elif version == 1:
        country_postal_codes = {}
        for country_code in zipgun_info['country_codes']:
            if country_code in country_postal_codes:
                raise ValueError('Duplicate entry found for {}'.format(
                    country_code))
            country_postal_codes[country_code] = SqliteDict(
                file_path, tablename='zg_{}'.format(country_code),
                journal_mode='OFF')
        zipgun_info.close()
    else:
        raise ValueError('Unknown data file version {}'.format(version))
    return country_postal_codes

def main(data_dir):
    print 'Loading data...'
    zg = Zipgun(data_dir, force_text=True)
    print 'Creating db...'
    persisted = SqliteDict(os.path.join(data_dir, DATA_FILE), autocommit=False)
    print 'Updating data...'
    persisted.update(zg.country_postal_codes)
    print 'Committing data...'
    persisted.commit()

 def test_reopen_conn(self):
     """Verify using a contextmanager that a connection can be reopened."""
     fname = norm_file('tests/db/sqlitedict-override-test.sqlite')
     db = SqliteDict(filename=fname)
     with db:
         db['key'] = 'value'
         db.commit()
     with db:
         db['key'] = 'value'
         db.commit()

    def test_tablenames(self):
        fname = norm_file('tests/db/tablenames-test-1.sqlite')
        SqliteDict(fname)
        self.assertEqual(SqliteDict.get_tablenames(fname), ['unnamed'])

        fname = norm_file('tests/db/tablenames-test-2.sqlite')
        with SqliteDict(fname,tablename='table1') as db1:
            self.assertEqual(SqliteDict.get_tablenames(fname), ['table1'])
        with SqliteDict(fname,tablename='table2') as db2:
            self.assertEqual(SqliteDict.get_tablenames(fname), ['table1','table2'])
        
        tablenames = SqliteDict.get_tablenames('tests/db/tablenames-test-2.sqlite')
        self.assertEqual(tablenames, ['table1','table2'])

def basic_usage():
    """SqliteDict引擎会将任意的value转换为
    """
    mydict = SqliteDict("test.sqlite", autocommit=True)
    mydict["integer_value"] = 1
    mydict["real_value"] = 2.2
    mydict["text_value"] = "abc"
    mydict["date_value"] = date.today()
    mydict["datetime_value"] = datetime.now()
    
    # if you don't use with SqliteDict("test.sqlite") as mydict: ...
    # you have to close the connection explicitly
    mydict.close() 

    def assertDatasetEquals(self, expected, tolerance):
        # Close the file to ensure it is written to disk.
        self.recorder.close()
        # self.recorder.out = None

        sentinel = object()

        db = SqliteDict( self.filename, self.tablename )


        for coord, expect in expected:
            iter_coord = format_iteration_coordinate(coord)

            groupings = (
                ("Parameters", expect[0]),
                ("Unknowns", expect[1]),
                ("Residuals", expect[2])
            )

            #### Need to get the record with the key of 'iter_coord'
            actual_group = db[iter_coord]
            timestamp = actual_group['timestamp']

            self.assertTrue(self.t0 <= timestamp and timestamp <= self.t1 )

            for label, values in groupings:
                actual = actual_group[label]
                # If len(actual) == len(expected) and actual <= expected, then
                # actual == expected.
                self.assertEqual(len(actual), len(values))
                for key, val in values:
                    found_val = actual.get(key, sentinel)
                    if found_val is sentinel:
                        self.fail("Did not find key '{0}'".format(key))
                    
                    if isinstance(found_val, _ByObjWrapper):
                        found_val = found_val.val

                    try:
                        assert_rel_error(self, found_val, val, tolerance)
                    except TypeError as error:
                        self.assertEqual(found_val, val)

            del db[iter_coord]
            ######## delete the record with the key 'iter_coord'

        # Having deleted all found values, the file should now be empty.
        ###### Need a way to get the number of records in the main table
        self.assertEqual(len(db), 0)

        db.close()

    def __init__(self, out, **sqlite_dict_args):
        super(SqliteRecorder, self).__init__()

        if MPI and MPI.COMM_WORLD.rank > 0 :
            self._open_close_sqlitedict = False
        else:
            self._open_close_sqlitedict = True

        if self._open_close_sqlitedict:
            sqlite_dict_args.setdefault('autocommit', True)
            self.out = SqliteDict(filename=out, flag='n', tablename='openmdao', **sqlite_dict_args)
            self.out_derivs = SqliteDict(filename=out, flag='w', tablename='openmdao_derivs', **sqlite_dict_args)

        else:
            self.out = None

    def assertDatasetEquals(self, expected, tolerance):
        # Close the file to ensure it is written to disk.
        self.recorder.close()
        # self.recorder.out = None

        sentinel = object()

        db = SqliteDict( self.filename, self.tablename )

        ###### Need a way to get a list of the group_names in the order in which they were written and put it in  a variable named order
        order = db['order']
        del db['order']

        for coord, expect in expected:
            iter_coord = format_iteration_coordinate(coord)

            self.assertEqual(order.pop(0), iter_coord)

            groupings = (
                ("Parameters", expect[0]),
                ("Unknowns", expect[1]),
                ("Residuals", expect[2])
            )

            #### Need to get the record with the key of 'iter_coord'
            actual_group = db[iter_coord]

            for label, values in groupings:
                actual = actual_group[label]
                # If len(actual) == len(expected) and actual <= expected, then
                # actual == expected.
                self.assertEqual(len(actual), len(values))
                for key, val in values:
                    found_val = actual.get(key, sentinel)
                    if found_val is sentinel:
                        self.fail("Did not find key '{0}'".format(key))
                    assert_rel_error(self, found_val, val, tolerance)
            del db[iter_coord]
            ######## delete the record with the key 'iter_coord'

        # Having deleted all found values, the file should now be empty.
        ###### Need a way to get the number of records in the main table
        self.assertEqual(len(db), 0)

        # As should the ordering.
        self.assertEqual(len(order), 0)

        db.close()

    def test_irregular_tablenames(self):
        """Irregular table names need to be quoted"""
        db = SqliteDict(':memory:', tablename='9nine')
        db['key'] = 'value'
        db.commit()
        self.assertEqual(db['key'], 'value')
        db.close()

        db = SqliteDict(':memory:', tablename='outer space')
        db['key'] = 'value'
        db.commit()
        self.assertEqual(db['key'], 'value')
        db.close()

        with self.assertRaisesRegexp(ValueError, r'^Invalid tablename '):
            SqliteDict(':memory:', '"')

    def __init__(self, basename, use_locks=True):
        """
        All data will be stored under directory `basename`. If there is a server
        there already, it will be loaded (resumed).

        The server object is stateless in RAM -- its state is defined entirely by its location.
        There is therefore no need to store the server object.
        """
        if not os.path.isdir(basename):
            raise ValueError("%r must be a writable directory" % basename)
        self.basename = basename
        self.lock_update = threading.RLock() if use_locks else gensim.utils.nocm
        try:
            self.fresh_index = SimIndex.load(self.location('index_fresh'))
        except:
            self.fresh_index = None
        try:
            self.opt_index = SimIndex.load(self.location('index_opt'))
        except:
            self.opt_index = None
        try:
            self.model = SimModel.load(self.location('model'))
        except:
            self.model = None
        self.payload = SqliteDict(self.location('payload'), autocommit=True, journal_mode=JOURNAL_MODE)
        # save the opened objects right back. this is not necessary and costs extra
        # time, but is cleaner when there are server location changes (see `check_moved`).
        self.flush(save_index=True, save_model=True, clear_buffer=True)
        logger.info("loaded %s" % self)

    def __init__(self, basename, use_locks=False):
        """
        All data will be stored under directory `basename`. If there is a server
        there already, it will be loaded (resumed).

        The server object is stateless in RAM -- its state is defined entirely by its location.
        There is therefore no need to store the server object.
        """
        if not os.path.isdir(basename):
            raise ValueError("%r must be a writable directory" % basename)
        self.basename = basename
        self.use_locks = use_locks
        self.lock_update = threading.RLock() if use_locks else gensim.utils.nocm
        try:
            self.fresh_index = SimIndex.load(self.location('index_fresh'))
        except:
            logger.debug("starting a new fresh index")
            self.fresh_index = None
        try:
            self.opt_index = SimIndex.load(self.location('index_opt'))
        except:
            logger.debug("starting a new optimized index")
            self.opt_index = None
        try:
            self.model = SimModel.load(self.location('model'))
        except:
            self.model = None
        self.payload = SqliteDict(self.location('payload'), autocommit=True, journal_mode=JOURNAL_MODE)
        self.flush(save_index=False, save_model=False, clear_buffer=True)
        logger.info("loaded %s" % self)

    def test_driver_records_model_viewer_data(self):
        size = 3

        prob = Problem(Group(), impl=impl)

        G1 = prob.root.add('G1', ParallelGroup())
        G1.add('P1', IndepVarComp('x', np.ones(size, float) * 1.0))
        G1.add('P2', IndepVarComp('x', np.ones(size, float) * 2.0))

        prob.root.add('C1', ABCDArrayComp(size))

        prob.root.connect('G1.P1.x', 'C1.a')
        prob.root.connect('G1.P2.x', 'C1.b')

        prob.driver.add_recorder(self.recorder)

        self.recorder.options['record_metadata'] = True
        prob.setup(check=False)

        prob.cleanup()

        # do some basic tests to make sure the model_viewer_data was recorded correctly
        if self.comm.rank == 0:
            db = SqliteDict(self.filename, self.tablename_metadata)
            model_viewer_data = db['model_viewer_data']
            tr = model_viewer_data['tree']
            self.assertEqual(set(['name', 'type', 'subsystem_type', 'children']), set(tr.keys()))

            names = []
            for ch1 in tr['children']:
                # each is an ordereddict
                names.append(ch1["name"] )
                for ch2 in ch1["children"]:
                    names.append(ch2["name"] )
                    if "children" in ch2:
                        for ch3 in ch2["children"]:
                            names.append(ch3["name"] )

            expected_names = ['G1', 'P1', 'x', 'P2', 'x', 'C1', 'a', 'b',
                        'in_string', 'in_list', 'c', 'd', 'out_string', 'out_list']

            self.assertEqual( sorted(expected_names), sorted(names))

            cl = model_viewer_data['connections_list']
            for c in cl:
                self.assertEqual(set(['src', 'tgt']), set(c.keys()))
            db.close()

    def test_recording_model_viewer_data(self):
        prob = Problem()
        prob.root = ConvergeDiverge()
        prob.driver.add_recorder(self.recorder)
        self.recorder.options["record_metadata"] = True
        prob.setup(check=False)
        prob.cleanup()  # closes recorders

        # do some basic tests to make sure the model_viewer_data was recorded
        db = SqliteDict(filename=self.filename, flag="r", tablename="metadata")
        model_viewer_data = db["model_viewer_data"]
        tr = model_viewer_data["tree"]
        self.assertEqual(set(["name", "type", "subsystem_type", "children"]), set(tr.keys()))
        cl = model_viewer_data["connections_list"]
        for c in cl:
            self.assertEqual(set(["src", "tgt"]), set(c.keys()))
        db.close()

    def test_recording_system_metadata(self):
        prob = Problem()
        prob.root = ConvergeDiverge()
        prob.root.add_metadata("string", "just a test")
        prob.root.add_metadata("ints", [1, 2, 3])
        prob.driver.add_recorder(self.recorder)
        self.recorder.options["record_metadata"] = True
        prob.setup(check=False)
        prob.cleanup()  # closes recorders

        # check the system metadata recording
        sqlite_metadata = SqliteDict(filename=self.filename, flag="r", tablename="metadata")
        system_metadata = sqlite_metadata["system_metadata"]
        self.assertEqual(len(system_metadata), 2)
        self.assertEqual(system_metadata["string"], "just a test")
        self.assertEqual(system_metadata["ints"], [1, 2, 3])
        sqlite_metadata.close()

 def test_1_theoretical_ion_space_step(self):
     print("test_1_theoretical_ion_space_step")
     ms_digest = MSDigestParameters.parse(self.protein_prospector_file)
     theo_ions = entry_point.generate_theoretical_ion_space(
         self.ms1_matching_output_file, self.glycosylation_sites_file,
         ms_digest.constant_modifications, ms_digest.variable_modifications,
         ms_digest.enzyme, self.num_procs)
     self.assertTrue(os.path.exists(theo_ions))
     self.theoretical_ion_space_file = theo_ions
     theoretical_ions = SqliteDict(theo_ions, tablename="theoretical_search_space")
     sequence_set = theoretical_ions.itervalues()
     peptide_sequences = [
         sequence.Sequence(s["Seq_with_mod"]) for s in sequence_set]
     peptide_mods = set()
     for seq in peptide_sequences:
         for resid, mod in seq:
             peptide_mods.update((m.rule for m in mod))
     print(peptide_mods)