def DBToData(dbName,tableName,user='sysdba',password='masterkey',dupCol=-1,
             what='*',where='',join='',pickleCol=-1,pickleClass=None,
             ensembleIds=None):
  """ constructs  an _MLData.MLDataSet_ from a database

    **Arguments**

      - dbName: the name of the database to be opened

      - tableName: the table name containing the data in the database

      - user: the user name to be used to connect to the database

      - password: the password to be used to connect to the database

      - dupCol: if nonzero specifies which column should be used to recognize
        duplicates.

    **Returns**

       an _MLData.MLDataSet_

    **Notes**

      - this uses Dbase.DataUtils functionality

  """
  conn = DbConnect(dbName,tableName,user,password)
  res = conn.GetData(fields=what,where=where,join=join,removeDups=dupCol,
                     forceList=1)
  nPts = len(res)
  vals = [None]*nPts
  ptNames = [None]*nPts
  classWorks=True
  for i in range(nPts):
    tmp = list(res[i])
    ptNames[i] = tmp.pop(0)
    if pickleCol>=0:
      if not pickleClass or not classWorks:
        tmp[pickleCol] = cPickle.loads(str(tmp[pickleCol]))
      else:
        try:
          tmp[pickleCol] = pickleClass(str(tmp[pickleCol]))
        except Exception:
          tmp[pickleCol] = cPickle.loads(str(tmp[pickleCol]))
          classWorks=False
      if ensembleIds:
        tmp[pickleCol] = BitUtils.ConstructEnsembleBV(tmp[pickleCol],ensembleIds)
    else:
      if ensembleIds:
        tmp = TakeEnsemble(tmp,ensembleIds,isDataVect=True)
    vals[i] = tmp  
  varNames = conn.GetColumnNames(join=join,what=what)
  data = MLData.MLDataSet(vals,varNames=varNames,ptNames=ptNames)
  return data

    def testPointPickles(self):
        pt = geom.Point3D(2.0,-3.0,1.0)
        pt2 = cPickle.loads(cPickle.dumps(pt))
        self.assertTrue(feq(pt.x,pt2.x,1e-6))
        self.assertTrue(feq(pt.y,pt2.y,1e-6))
        self.assertTrue(feq(pt.z,pt2.z,1e-6))

        pt = geom.Point2D(2.0,-4.0)
        pt2 = cPickle.loads(cPickle.dumps(pt))
        self.assertTrue(feq(pt.x,pt2.x,1e-6))
        self.assertTrue(feq(pt.y,pt2.y,1e-6))

 def testPkl2(self):
     """ further pickle tests """
     smis = self.bigSmiList
     for smi in smis:
         m = Chem.MolFromSmiles(smi)
         newM1 = cPickle.loads(cPickle.dumps(m))
         newM2 = cPickle.loads(cPickle.dumps(newM1))
         oldSmi = Chem.MolToSmiles(newM1)
         newSmi = Chem.MolToSmiles(newM2)
         assert newM1.GetNumAtoms() == m.GetNumAtoms(), "num atoms comparison failed"
         assert newM2.GetNumAtoms() == m.GetNumAtoms(), "num atoms comparison failed"
         assert oldSmi == newSmi, "string compare failed: %s != %s" % (oldSmi, newSmi)

  def test2CatStringPickle(self):
    self._fillCat(self.smiList2)

    # test non-binary pickle:
    cat2 = cPickle.loads(cPickle.dumps(self.fragCat))
    assert cat2.GetNumEntries()==21
    assert cat2.GetFPLength()==21
    self._testBits(cat2)

    # test binary pickle:
    cat2 = cPickle.loads(cPickle.dumps(self.fragCat,1))
    assert cat2.GetNumEntries()==21
    assert cat2.GetFPLength()==21
    self._testBits(cat2)

    def testPickle(self):
      ffeat = ChemicalFeatures.FreeChemicalFeature("HBondDonor", "HBondDonor1", geom.Point3D(1.0, 2.0, 3.0),123)
      pkl = cPickle.dumps(ffeat)
      ffeat2 = cPickle.loads(pkl, encoding='bytes')
      self.assertTrue(ffeat2.GetId()==ffeat.GetId());      
      self.assertTrue(ffeat2.GetFamily()==ffeat.GetFamily())
      self.assertTrue(ffeat2.GetType()==ffeat.GetType())
      self.assertTrue(ptFeq(ffeat2.GetPos(),ffeat.GetPos()))

      # Check that the old pickled versions have not been broken        
      inF = open(os.path.join(RDConfig.RDBaseDir,
                              'Code/ChemicalFeatures/Wrap/testData/feat.pkl'),'rb')
      ffeat2=cPickle.load(inF, encoding='bytes')
      # this version (1.0) does not have an id in the byte stream 
      self.assertTrue(ffeat2.GetFamily()==ffeat.GetFamily())
      self.assertTrue(ffeat2.GetType()==ffeat.GetType())
      self.assertTrue(ptFeq(ffeat2.GetPos(),ffeat.GetPos()))
        
      # Test the new version also has the id and works as expected
      
      # uncomment the following to generate (overrwrite) new version of pickled
      # data file
      #cPickle.dump(ffeat,file(os.path.join(RDConfig.RDBaseDir, 'Code/ChemicalFeatures/Wrap/testData/featv2.pkl'),'wb+'))
      inF = open(os.path.join(RDConfig.RDBaseDir,
                              'Code/ChemicalFeatures/Wrap/testData/featv2.pkl'),'rb')
      ffeat2=cPickle.load(inF, encoding='bytes')
      self.assertTrue(ffeat2.GetId()==ffeat.GetId());
      self.assertTrue(ffeat2.GetFamily()==ffeat.GetFamily())
      self.assertTrue(ffeat2.GetType()==ffeat.GetType())
      self.assertTrue(ptFeq(ffeat2.GetPos(),ffeat.GetPos()))

  def test3Pickle2(self):
    """

    """
    l=1<<21
    v1 = ds.IntSparseIntVect(l)
    self.assertRaises(IndexError,lambda:v1[l+1])
    v1[0]=1
    v1[2]=2
    v1[1<<12]=3
    self.assertTrue(v1==v1)

    v2=  cPickle.loads(cPickle.dumps(v1))
    self.assertTrue(v2==v1)
    
    v3=  ds.IntSparseIntVect(v2.ToBinary())
    self.assertTrue(v2==v3)
    self.assertTrue(v1==v3)

    #cPickle.dump(v1,file('isiv.pkl','wb+'))
    with open(
      os.path.join(RDConfig.RDBaseDir, 
                   'Code/DataStructs/Wrap/testData/isiv.pkl'),
      'rb'
      ) as f:
      v3 = cPickle.load(f)
      self.assertTrue(v3==v1)

    def next(self):
      curs = self.cursor
      if not curs or \
             curs.closed or \
             curs.conn is None or \
             curs.res is None or \
             (curs.res.resultType != sql.RESULT_DQL and curs.closed is None):
        raise StopIteration
      if not self._first:
        res = curs.conn.conn.query('fetch 1 from "%s"'%self.cursor.name)

        if res.ntuples == 0:
          raise StopIteration
        else:
          if res.nfields < 2:
            raise ValueError,'bad result: %s'%str(res)
          t = [res.getvalue(0,x) for x in range(res.nfields)]
          val = t[self._pickleCol]
      else:
        t = curs.fetchone()
        val = str(t[self._pickleCol])
        self._first = 0
      if self._depickle:
        if not self._klass:
          fp = cPickle.loads(val)
        else:
          fp = self._klass(val)
        fields = list(t)
        del fields[self._pickleCol]
        fp._fieldsFromDb = fields
      else:
        fp = list(t)
      return fp

    def __getitem__(self,idx):
      if self.res is None:
        self.cursor.execute(self.cmd)
        self._first = self.cursor.fetchone()
        self._validate()
        self.res = self.cursor.conn.conn.query('fetch all from "%s"'%self.cursor.name)
        self.rowCount = self.res.ntuples+1
        self.idx=0
        if self.res.nfields < 2:
          raise ValueError,'bad query result'%str(res)

      if idx < 0:
        idx = self.rowCount+idx
      if idx<0 or (idx >= 0 and idx >= self.rowCount):
        raise IndexError
      if idx==0:
        val = str(self._first[self._pickleCol])
        t = list(self._first)
      else:
        val = self.res.getvalue(self.idx-1,self._pickleCol)
        t = [self.res.getvalue(self.idx-1,x) for x in range(self.res.nfields)]
      if self._depickle:
        try:
          fp = cPickle.loads(val)
        except:
          import logging
          del t[self._pickleCol]
          logging.exception('Depickling failure in row: %s'%str(t))
          raise
        del t[self._pickleCol]
        fp._fieldsFromDb = t
      else:
        fp = t
      return fp

  def test3Pickle2(self):
    """

    """
    l=1<<21
    v1 = ds.IntSparseIntVect(l)
    self.assertRaises(IndexError,lambda:v1[l+1])
    v1[0]=1
    v1[2]=2
    v1[1<<12]=3
    self.assertTrue(v1==v1)

    v2=  cPickle.loads(cPickle.dumps(v1))
    self.assertTrue(v2==v1)
    
    v3=  ds.IntSparseIntVect(v2.ToBinary())
    self.assertTrue(v2==v3)
    self.assertTrue(v1==v3)

    #cPickle.dump(v1,file('isiv.pkl','wb+'))
    with open(
      os.path.join(RDConfig.RDBaseDir, 
                   'Code/DataStructs/Wrap/testData/isiv.pkl'),
      'r'
      ) as tf:
      buf = tf.read().replace('\r\n', '\n').encode('utf-8')
      tf.close()
    with io.BytesIO(buf) as f:
      v3 = cPickle.load(f)
      self.assertTrue(v3==v1)

 def test4Serialize(self):
   with open(self.smiName, 'r') as smiF:
     smiLines = smiF.readlines()
   fparams = FragmentCatalog.FragCatParams(1, 6, self.fName)
   fcat = FragmentCatalog.FragCatalog(fparams)
   fgen = FragmentCatalog.FragCatGenerator()
   suppl = Chem.SmilesMolSupplier(self.smiName, " ", 0, 1, 0)
   smiles = []
   for mol in suppl:
     nent = fgen.AddFragsFromMol(mol, fcat)
     smiles.append(Chem.MolToSmiles(mol))
   self.assertEqual(fcat.GetNumEntries(), 21)
   self.assertEqual(fcat.GetFPLength(), 21)
   pkl = cPickle.dumps(fcat)
   fcat2 = cPickle.loads(pkl)
   self.assertEqual(fcat2.GetNumEntries(), 21)
   self.assertEqual(fcat2.GetFPLength(), 21)
   fpgen = FragmentCatalog.FragFPGenerator()
   for i in range(len(smiles)):
     smi = smiles[i]
     mol = Chem.MolFromSmiles(smi)
     fp1 = fpgen.GetFPForMol(mol, fcat)
     fp2 = fpgen.GetFPForMol(mol, fcat2)
     self.assertEqual(fp1.GetNumOnBits(), fp2.GetNumOnBits())
     obl1 = fp1.GetOnBits()
     obl2 = fp2.GetOnBits()
     self.assertEqual(tuple(obl1), tuple(obl2))

 def testPkl1(self):
     " testing single molecule pickle "
     m = Chem.MolFromSmiles("CCOC")
     outS = Chem.MolToSmiles(m)
     m2 = cPickle.loads(cPickle.dumps(m))
     outS2 = Chem.MolToSmiles(m2)
     assert outS == outS2, "bad pickle: %s != %s" % (outS, outS2)

def getFingerprints(conn) :
    data = conn.GetData(table='signatures', fields='mol_name,fingerprint')
    fpMap = {}
    for dat in data :
        pkl = str(dat[1])
        sbv = pickle.loads(pkl)
        fpMap[dat[0]] = sbv
    return fpMap

 def _testPkl10(self):
   " testing 5k molecule pickles "
   inLines = open('%s/NCI/first_5K.smi'%(RDConfig.RDDataDir),'r').readlines()
   smis = []
   for line in inLines:
     smis.append(line.split('\t')[0])
   for smi in smis:
     m = Chem.MolFromSmiles(smi)
     newM1 = cPickle.loads(cPickle.dumps(m))
     newSmi1 = Chem.MolToSmiles(newM1)
     newM2 = cPickle.loads(cPickle.dumps(newM1))
     newSmi2 = Chem.MolToSmiles(newM2)
     assert newM1.GetNumAtoms()==m.GetNumAtoms(),'num atoms comparison failed'
     assert newM2.GetNumAtoms()==m.GetNumAtoms(),'num atoms comparison failed'
     assert len(newSmi1)>0,'empty smi1'
     assert len(newSmi2)>0,'empty smi2'
     assert newSmi1==newSmi2,'string compare failed:\n%s\n\t!=\n%s\norig smiles:\n%s'%(newSmi1,newSmi2,smi)

  def test6PickleEquals(self):
    " testing pickled tree equals "
    self._readyTree()
    pkl = cPickle.dumps(self.baseTree)
    oTree = cPickle.loads(pkl)

    assert oTree == self.baseTree, 'Pickle inequality test failed'

    self.baseTree.PruneChild(self.baseTree.GetChildren()[0])
    assert oTree != self.baseTree, 'Pickle inequality test failed (bad Tree.__cmp__)'

  def test2ExplicitPickle(self):
    nbits = 10000
    bv1 = DataStructs.ExplicitBitVect(nbits)
    for i in range(1000):
      x = random.randrange(0, nbits)
      bv1.SetBit(x)

    pkl = pickle.dumps(bv1, 1)
    bv2 = pickle.loads(pkl)
    for i in range(nbits):
      assert bv1[i] == bv2[i]

def GetComposites(details):
  res = []
  if details.persistTblName and details.inNote:
    conn = DbConnect(details.dbName,details.persistTblName)
    mdls = conn.GetData(fields='MODEL',where="where note='%s'"%(details.inNote))
    for row in mdls:
      rawD = row[0]
      res.append(cPickle.loads(str(rawD)))
  elif details.composFileName:
    res.append(cPickle.load(open(details.composFileName,'rb')))
  return res

def DepickleFP(pkl,similarityMethod):
    if not isinstance(pkl,(bytes,str)):
      pkl = str(pkl)
    try:
        klass=similarityMethods[similarityMethod]
        fp = klass(pkl)
    except Exception:
        import traceback
        traceback.print_exc()
        fp = cPickle.loads(pkl)
    return fp

 def testMQNDetails(self):
   refFile = os.path.join(RDConfig.RDCodeDir,'Chem','test_data','MQNs_regress.pkl')
   with open(refFile,'rb') as inf:
     pkl = inf.read()
   refData  = cPickle.loads(pkl,encoding='bytes')
   fn = os.path.join(RDConfig.RDCodeDir,'Chem','test_data','aromat_regress.txt')
   ms = [x for x in Chem.SmilesMolSupplier(fn,delimiter='\t')]
   for i,m in enumerate(ms):
     mqns = rdMolDescriptors.MQNs_(m) 
     if mqns!=refData[i][1]:
       indices=[(j,x,y) for j,x,y in zip(range(len(mqns)),mqns,refData[i][1]) if x!=y]
       print(Chem.MolToSmiles(m),indices)
     self.assertEqual(mqns,refData[i][1])

 def test3Pickles(self):
   #outF = file('../testData/rvvs.pkl','wb+')
   with open(os.path.join(RDConfig.RDBaseDir,
                  'Code/DataStructs/Wrap/testData/rvvs.pkl'),
     'rb') as inF:
       v1 = ds.RealValueVect(30)
       for i in range(15):
           v1[2*i] = 1.3
       v2 = cPickle.loads(cPickle.dumps(v1))
       self.assertAlmostEqual(ds.ComputeL1Norm(v1, v2), 0)
       #cPickle.dump(v1,outF)
       v2=cPickle.load(inF, encoding='bytes')
       self.assertAlmostEqual(ds.ComputeL1Norm(v1, v2), 0)
       self.assertAlmostEqual(v1.GetTotalVal(), v2.GetTotalVal())
       self.failUnless(v2.GetTotalVal()!=0)

 def setUp(self):
   self.dataset = dict()
   self.dataset_inchi = dict()
   inf = gzip.open(
     os.path.join(RDConfig.RDCodeDir, 'Chem/test_data', 'pubchem-hard-set.sdf.gz'), 'r')
   self.dataset['problematic'] = ForwardSDMolSupplier(inf, sanitize=False, removeHs=False)
   with open(os.path.join(RDConfig.RDCodeDir, 'Chem/test_data', 'pubchem-hard-set.inchi'),
             'r') as intF:
     buf = intF.read().replace('\r\n', '\n').encode('latin1')
     intF.close()
   with io.BytesIO(buf) as inF:
     pkl = inF.read()
   self.dataset_inchi['problematic'] = loads(pkl, encoding='latin1')
   # disable logging
   DisableLog('rdApp.warning')