def _restimateCTF(self, ctfId):
        """ Run ctffind3 with required parameters """

        ctfModel = self.recalculateSet[ctfId]
        mic = ctfModel.getMicrograph()
        micFn = mic.getFileName()
        micDir = self._getMicrographDir(mic)
        
        out = self._getCtfOutPath(micDir)
        psdFile = self._getPsdPath(micDir)
        
        pwutils.cleanPath(out)
        micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, "mrc"))
        em.ImageHandler().convert(micFn, micFnMrc, em.DT_FLOAT)

        # Update _params dictionary
        self._prepareRecalCommand(ctfModel)
        self._params['micFn'] = micFnMrc
        self._params['micDir'] = micDir
        self._params['ctffindOut'] = out
        self._params['ctffindPSD'] = psdFile
        
        pwutils.cleanPath(psdFile)
        try:
            self.runJob(self._program, self._args % self._params)
        except Exception, ex:
            print >> sys.stderr, "ctffind has failed with micrograph %s" % micFnMrc

    def show(self, form):
        prot = form.protocol
        micSet = prot.getInputMicrographs()

        if not micSet:
            print 'must specify input micrographs'
            return

        project = prot.getProject()
        micfn = micSet.getFileName()

        # Prepare a temporary folder to convert some input files
        # and put some of the intermediate result files
        coordsDir = project.getTmpPath(micSet.getName())
        pwutils.cleanPath(coordsDir)
        pwutils.makePath(coordsDir)
        prot.convertInputs(coordsDir)

        pickerConfig = os.path.join(coordsDir, 'picker.conf')
        f = open(pickerConfig, "w")

        pickScript = pw.join('em', 'packages', 'igbmc', 'run_gempicker.py')

        pickCmd = prot._getPickArgs(threshold=False, workingDir=coordsDir)[0]
        convertCmd = pw.join('apps', 'pw_convert.py')

        args = {
                "pickScript": pickScript,
                "pickCmd": pickCmd,
                "convertCmd": convertCmd,
                'coordsDir': coordsDir,
                'micsSqlite': micSet.getFileName(),
                'thresholdLow': prot.thresholdLow,
                'thresholdHigh': prot.thresholdHigh,
                "useGPU": prot.useGPU
          }

        f.write("""
        parameters = thresholdLow,thresholdHigh
        thresholdLow.value =  %(thresholdLow)s
        thresholdLow.label = Threshold Low
        thresholdLow.help = Low value cut-off
        thresholdHigh.value =  %(thresholdHigh)s
        thresholdHigh.label = Threshold High
        thresholdHigh.help = High value cut-off
        autopickCommand = %(pickScript)s %%(micrograph) %(coordsDir)s %(useGPU)s %(pickCmd)s --thresh=%%(thresholdLow) --threshHigh=%%(thresholdHigh)
        convertCommand = %(convertCmd)s --coordinates --from gempicker --to xmipp --input  %(micsSqlite)s --output %(coordsDir)s
        """ % args)

        f.close()

        process = CoordinatesObjectView(project, micfn, coordsDir, prot,
                                        mode=CoordinatesObjectView.MODE_AUTOMATIC,
                                        pickerProps=pickerConfig).show()
        process.wait()
        myprops = pwutils.readProperties(pickerConfig)

        if myprops['applyChanges'] == 'true':
            form.setVar('thresholdLow', myprops['thresholdLow.value'])
            form.setVar('thresholdHigh', myprops['thresholdHigh.value'])

def runProjectionMatching(self, iterN, refN, args, **kwargs):
    """ Loop over all CTF groups and launch a projection matching for each one.
    Note: Iterate ctf groups in reverse order to have same order as 
          in add_to docfiles from angular_class_average. #FIXME: check why reverse order is needed
    """
    projMatchRootName = self._getFileName('projMatchRootNames', iter=iterN, ref=refN)
    refname = self._getFileName('projectLibraryStk', iter=iterN, ref=refN)
    
    numberOfCtfGroups = self.numberOfCtfGroups.get()
#     ctfGroupName = self._getPath(self.ctfGroupDirectory, '%(ctfGroupRootName)s')
    #remove output metadata
    cleanPath(projMatchRootName)
    
    for ctfN in reversed(list(self.allCtfGroups())):
        self._log.info('CTF group: %d/%d' % (ctfN, numberOfCtfGroups))
        ctfArgs = ' -i %(inputdocfile)s -o %(outputname)s --ref %(refname)s'        
        
        inputdocfile = self._getBlockFileName(ctfBlockName, ctfN, self.docFileInputAngles[iterN-1])
        outputname = self._getBlockFileName(ctfBlockName, ctfN, projMatchRootName)
        baseTxtFile = removeExt(refname)
        neighbFile = baseTxtFile + '_sampling.xmd'
        cleanPath(neighbFile)
        neighbFileb = baseTxtFile + '_group' + str(ctfN).zfill(self.FILENAMENUMBERLENGTH) + '_sampling.xmd'
        copyFile(neighbFileb, neighbFile)
        print "copied file ", neighbFileb, "to", neighbFile
        if self.doCTFCorrection and self._referenceIsCtfCorrected[iterN]:
            ctfArgs += ' --ctf %s' % self._getBlockFileName('', ctfN, self._getFileName('stackCTFs'))
    
        progArgs = ctfArgs % locals() + args
        self.runJob('xmipp_angular_projection_matching', progArgs, **kwargs)

def scipion_split_particle_stacks(inputStar, inputStack, output, filename_prefix, deleteStack):
    """ Read a STAR file with particles and write as individual images.
    If a stack of images is given, use these instead of the images from the STAR file.
    Also write a new STAR file pointing to these images.
    This function requires that the script is run within Scipion Python environment. """

    import pyworkflow.utils as pwutils
    from pyworkflow.em import ImageHandler
    ih = ImageHandler()
    md = MetaData(inputStar)
    md.addLabels('rlnOriginalName')

    # Initialize progress bar
    progressbar = ProgressBar(width=60, total=len(md))

    for i, particle in enumerate(md, start=1):
        outputImageName = '%s/%s_%06d.mrc' % (output, filename_prefix, i)

        if inputStack:
            ih.convert((i, inputStack), outputImageName )
            particle.rlnOriginalName = '%s/%[email protected]%s' %(output, i, inputStack)
        else:
            ih.convert(particle.rlnImageName, outputImageName)
            particle.rlnOriginalName = particle.rlnImageName

        particle.rlnImageName = outputImageName

        progressbar.notify()

    print("\n")
    md.write("%s/%s.star" % (output, filename_prefix))

    if inputStack and deleteStack:
        pwutils.cleanPath(inputStack)

    def extractMicrographStep(self, micKey, *args):
        """ Step function that will be common for all extraction protocols.
        It will take an id and will grab the micrograph from a micDict map.
        The micrograph will be passed as input to the _extractMicrograph
        function.
        """
        # Retrieve the corresponding micrograph with this key and the
        # associated list of coordinates
        mic = self.micDict[micKey]

        micDoneFn = self._getMicDone(mic)
        micFn = mic.getFileName()

        if self.isContinued() and os.path.exists(micDoneFn):
            self.info("Skipping micrograph: %s, seems to be done" % micFn)
            return

        coordList = self.coordDict[mic.getObjId()]
        self._convertCoordinates(mic, coordList)

        # Clean old finished files
        pwutils.cleanPath(micDoneFn)

        self.info("Extracting micrograph: %s " % micFn)
        self._extractMicrograph(mic, *args)

        # Mark this mic as finished
        open(micDoneFn, 'w').close()

 def nestedFlatDb(self): 
     fn = 'classes_flat.sqlite'
     cleanPath(fn)
     
     images = SetOfImages()
     images.setSamplingRate(1.2)
     classes2DSet = SetOfClasses2D(filename=fn)
     classes2DSet.setImages(images)
 
     for ref in range(1, 11):
         print "class: ", ref
         class2D = Class2D()
         class2D.setObjId(ref)
         print "append class to set, ref=", ref
         classes2DSet.append(class2D)
         avg = Particle()
         avg.setLocation(ref, 'averages.stk')
         print "   populating class "
         for i in range(1, 101):         
             img = Particle()
             img.setSamplingRate(5.3)
             class2D.append(img)
                 
         print "   writing class "
         class2D.write()
         print "   append avg"
         averages.append(avg)
     
     classes2DSet.write()

def exportData(emxDir, inputSet, ctfSet=None,
               xmlFile='data.emx', binaryFile=None,
               doConvert=True):
    """ Export micrographs, coordinates or particles to  EMX format. """
    pwutils.cleanPath(emxDir)
    pwutils.makePath(emxDir) 
    emxData = emxlib.EmxData()
    micSet=None
    
    if isinstance(inputSet, SetOfMicrographs):
        _micrographsToEmx(emxData, inputSet, emxDir, ctfSet, writeData=True)
        
    elif isinstance(inputSet, SetOfCoordinates):
        micSet = inputSet.getMicrographs()
        _micrographsToEmx(emxData, micSet, emxDir, ctfSet)

        _particlesToEmx(emxData, inputSet, micSet, writeImages=False)
#        _particlesToEmx(emxData, inputSet, None, micSet, doConvert=doConvert)
        
    elif isinstance(inputSet, SetOfParticles):
        if inputSet.hasCoordinates():
            micSet = inputSet.getCoordinates().getMicrographs()
            _micrographsToEmx(emxData, micSet, emxDir, writeData=False)

        kwargs = {'writeImages': True}
        
        if binaryFile is None:
            kwargs['imagesPrefix'] = emxDir
        else:
            kwargs['imagesStack'] = join(emxDir, binaryFile)
            
        _particlesToEmx(emxData, inputSet, micSet, **kwargs)

    fnXml = join(emxDir, xmlFile)
    emxData.write(fnXml)

 def decompress(program, args, ext, nExt):
     movieFolder = self._getTmpPath()
     movieName = basename(movie.getFileName())
     movieTmpLink = join(movieFolder, movieName)
     pwutils.cleanPath(movieTmpLink)
     pwutils.createAbsLink(os.path.abspath(movieFn), movieTmpLink)
     self.runJob(program, args % movieName, cwd=movieFolder)
     dimMovie.setFileName(movieTmpLink.replace(ext, nExt))

def writeVolumesSqlite(volsXml, volsSqlite, **kwargs):
    """ Convert a volume list from PyTom xml format to Scipion sqlite file.
    If the volSqlite exists, it will be deleted.
    """
    pwutils.cleanPath(volsSqlite)
    volSet = em.SetOfVolumes(filename=volsSqlite)    
    readSetOfVolumes(volsXml, volSet, **kwargs)
    volSet.write()

    def runCase(self, args, mpi=0, changeDir=False, 
                preruns=None, postruns=None, validate=None,
                outputs=None, random=False):
        # Retrieve the correct case number from the test name id
        # We asumme here that 'test_caseXXX' should be in the name
        caseId = unittest.TestCase.id(self)
        if not 'test_case' in caseId:
            raise Exception("'test_case' string should be in the test function name followed by a number")
        _counter = int(caseId.split('test_case')[1])

        self._testDir = self.dataset.getPath()
        self.outputDir = os.path.join('tmpLink', '%s_%02d' % (self.program, _counter))
        self.outputDirAbs = os.path.join(self._testDir, self.outputDir)
        self.goldDir = os.path.join(self._testDir, 'gold', '%s_%02d' % (self.program, _counter))
        
        # Clean and create the program output folder if not exists
        pwutils.cleanPath(self.outputDirAbs)
        pwutils.makePath(self.outputDirAbs)
        
        # Change to tests root folder (self._testDir)
        cwd = os.getcwd()
        os.chdir(self._testDir)
        
        if preruns:
            self._runCommands(preruns, 'preruns')
            
        if mpi:
            cmd = "mpirun -np %d `which %s`" % (mpi, self.program)
        else:
            cmd = self.program
        
        args = self._parseArgs(args)
        
        if changeDir:
            cmd = "cd %s ; %s %s > stdout.txt 2> stderr.txt" % (self.outputDir, cmd, args)
        else:
            cmd = "%s %s > %s/stdout.txt 2> %s/stderr.txt" % (cmd, args, self.outputDir, self.outputDir)
        print "    Command: "
        print "       ", pwutils.green(cmd)
            
        #run the test itself
        command = Command(cmd, env=self.env)
        self._command = command
        try:
            command.run(timeout=self._timeout)
        except KeyboardInterrupt:
            command.terminate()
        
        if postruns:
            self._runCommands(postruns, 'postruns')
            
        if outputs:
            self._checkOutputs(outputs,random)
            
        if validate:
            validate()
            
        os.chdir(cwd)

 def close(self, e=None):
     try:
         print "Writing protocols to: ", jsonFn
         proj.getRunsGraph() # Build project runs graph
         proj.exportProtocols(proj.getRuns(), jsonFn)
         print "Deleting temporary folder: ", customUserData
         pwutils.cleanPath(customUserData)
     except Exception, ex:
         print "Error saving the workflow: ", ex

 def filterStep(self, args):
     """ Apply the selected filter to particles. 
     Create the set of particles.
     """
     particlesStk = self._getPath('particles.spi')
     tmpStk = particlesStk.replace('.spi', '_tmp.spi')
     self.runJob('bfilter', args + ' %s %s' % (particlesStk, tmpStk))
     pwutils.moveFile(tmpStk, particlesStk.replace('.spi', '.stk')) # just we prefer stk as stack of spider images
     pwutils.cleanPath(particlesStk)

 def _writeXmippCoords(self, coordSet):
     micSet = self.getInputMicrographs()
     coordPath = self._getTmpPath('xmipp_coordinates')
     pwutils.cleanPath(coordPath)
     pwutils.makePath(coordPath)
     import pyworkflow.em.packages.xmipp3 as xmipp3
     micPath = micSet.getFileName()
     xmipp3.writeSetOfCoordinates(coordPath, coordSet, ismanual=False)
     return micPath, coordPath

 def filterStep(self, args):
     """ Apply the selected filter to particles. 
     Create the set of particles.
     """
     particlesStk = self._getPath("particles.spi")
     tmpStk = particlesStk.replace(".spi", "_tmp.spi")
     self.runJob("bfilter", args + " %s %s" % (particlesStk, tmpStk))
     pwutils.moveFile(tmpStk, particlesStk.replace(".spi", ".stk"))  # just we prefer stk as stack of spider images
     pwutils.cleanPath(particlesStk)

 def _writeXmippCoords(self, coordSet):
     micSet = self.getInputMicrographs()
     coordPath = self._getTmpPath('xmipp_coordinates')
     pwutils.cleanPath(coordPath)
     pwutils.makePath(coordPath)
     import pyworkflow.em.packages.xmipp3 as xmipp3
     micPath = os.path.join(coordPath, 'micrographs.xmd')
     xmipp3.writeSetOfMicrographs(micSet, micPath)
     xmipp3.writeSetOfCoordinates(coordPath, coordSet)
     return micPath, coordPath

    def test_createEmptyFile(self):
        x, y, z, n = 10, 10, 1, 100
        img = Image()

        for ext in ['stk', 'mrcs']:
            imgPath = testFile('empty_100.%s' % ext)
            createEmptyFile(imgPath, x, y, z, n)
            img.read(imgPath, HEADER)
            self.assertEqual(img.getDimensions(), (x, y, z, n))
            pwutils.cleanPath(imgPath)

    def show(self, form):
        autopickProt = form.protocol
        micSet = autopickProt.getInputMicrographs()
        if not micSet:
            print 'must specify input micrographs'
            return
        project = autopickProt.getProject()
        micfn = micSet.getFileName()
        coordsDir = project.getTmpPath(micSet.getName())
        cleanPath(coordsDir)
        makePath(coordsDir)
        
        pickerProps = os.path.join(coordsDir, 'picker.conf')
        f = open(pickerProps, "w")
        params = ['boxSize', 'lowerThreshold', 'higherThreshold', 'gaussWidth']
        args = {
          "params": ','.join(params),
          "preprocess" : os.path.join(os.environ['SCIPION_HOME'], "scipion sxprocess.py"),
          "picker" : os.path.join(os.environ['SCIPION_HOME'], "scipion e2boxer.py"),
          "convert" : os.path.join(os.environ['SCIPION_HOME'], os.path.join('pyworkflow','apps', 'pw_convert.py')),
          'coordsDir':coordsDir,
          'micsSqlite': micSet.getFileName(),
          "boxSize": autopickProt.boxSize,
          "lowerThreshold": autopickProt.lowerThreshold,
          "higherThreshold": autopickProt.higherThreshold,
          "gaussWidth": autopickProt.gaussWidth,
          "extraParams":autopickProt.extraParams
          }


        f.write("""
        parameters = %(params)s
        boxSize.value = %(boxSize)s
        boxSize.label = Box Size
        boxSize.help = some help
        lowerThreshold.value =  %(lowerThreshold)s
        lowerThreshold.label = Lower Threshold
        lowerThreshold.help = some help
        higherThreshold.help = some help
        higherThreshold.value =  %(higherThreshold)s
        higherThreshold.label = Higher Threshold
        gaussWidth.help = some help
        gaussWidth.value =  %(gaussWidth)s
        gaussWidth.label = Gauss Width
        runDir = %(coordsDir)s
        preprocessCommand = %(preprocess)s demoparms --makedb=thr_low=%%(lowerThreshold):thr_hi=%%(higherThreshold):boxsize=%%(boxSize):gauss_width=%%(gaussWidth):%(extraParams)s
        autopickCommand = %(picker)s --gauss_autoboxer=demoparms --write_dbbox --boxsize=%%(boxSize) --norm=normalize.ramp.normvar %%(micrograph) 
        convertCommand = %(convert)s --coordinates --from eman2 --to xmipp --input  %(micsSqlite)s --output %(coordsDir)s
        """ % args)
        f.close()
        process = CoordinatesObjectView(project, micfn, coordsDir, autopickProt, pickerProps=pickerProps).show()
        process.wait()
        myprops = readProperties(pickerProps)
        for param in params:
            form.setVar(param, myprops[param + '.value'])

    def _fixMovie(self, movie):
        if self.doSaveMovie and self.useMotioncor2 and self._isOutStackSupport():
            outputMicFn = self._getExtraPath(self._getOutputMicName(movie))
            outputMovieFn = self._getExtraPath(self._getOutputMovieName(movie))
            movieFn = outputMicFn.replace('_aligned_mic.mrc',
                                          '_aligned_mic_Stk.mrc')
            pwutils.moveFile(movieFn, outputMovieFn)

        if self.useMotioncor2 and not self.doSaveUnweightedMic:
            fnToDelete = self._getExtraPath(self._getOutputMicName(movie))
            pwutils.cleanPath(fnToDelete)

 def writeZeroShifts(self, movie):
     # TODO: find another way to do this
     shiftsMd = self._getTmpPath('zero_shifts.xmd')
     pwutils.cleanPath(shiftsMd)
     xshifts = [0] * movie.getNumberOfFrames()
     yshifts = xshifts
     alignment = MovieAlignment(first=1, last=movie.getNumberOfFrames(),
                                xshifts=xshifts, yshifts=yshifts)
     roiList = [0, 0, 0, 0]
     alignment.setRoi(roiList)
     movie.setAlignment(alignment)
     writeShiftsMovieAlignment(movie, shiftsMd,
                               1, movie.getNumberOfFrames())
     return shiftsMd

def main():
    parser = argparse.ArgumentParser(prog='Scipion Convert')
    parser.add_argument('--coordinates', help='Convert coordinates', action="store_true")
    parser.add_argument('--fromType', help='Convert from input type')
    parser.add_argument('--toType', help='Convert to output type')
    parser.add_argument('--input', help='Input file or folder')
    parser.add_argument('--output', help='Output file or folder')
    parser.add_argument('--extra', help='To add extra parameters')

    args = parser.parse_args()
    fromType = args.fromType
    toType = args.toType
    input = args.input
    output = args.output

    if args.coordinates:
        micSet = loadSetFromDb(input)
        outputDir = output
        coordsfn = os.path.join(outputDir, 'coordinates.sqlite')
        cleanPath(coordsfn)
        coordSet = SetOfCoordinates(filename=coordsfn)
        coordSet.setMicrographs(micSet)

        if fromType == 'eman2':
            if toType == 'xmipp': 
                from pyworkflow.em.packages.eman2.convert import readSetOfCoordinates
        elif fromType == 'dogpicker':
            if toType == 'xmipp': 
                from pyworkflow.em.packages.appion.convert import readSetOfCoordinates
        elif fromType == 'relion':
            if toType == 'xmipp':
                def readSetOfCoordinates(outputDir, micSet, coordSet):
                    from pyworkflow.em.packages.relion.convert import readSetOfCoordinates
                    inputCoords = args.extra
                    starFiles = [os.path.join(inputCoords,
                                              pwutils.removeBaseExt(mic.getFileName())
                                              + '_autopick.star') for mic in micSet]
                    readSetOfCoordinates(coordSet, starFiles)
        elif fromType == 'gautomatch':
            if toType == 'xmipp':
                from pyworkflow.em.packages.gautomatch.convert import readSetOfCoordinates
        elif fromType == 'gempicker':
            if toType == 'xmipp':
                from pyworkflow.em.packages.igbmc.convert import readSetOfCoordinates
        else:
            raise Exception('Unknown coordinates type: %s' % fromType)

        readSetOfCoordinates(outputDir, micSet, coordSet)
        from pyworkflow.em.packages.xmipp3.convert import writeSetOfCoordinatesWithState
        writeSetOfCoordinatesWithState(outputDir, coordSet, state='Automatic')