def test_utfcrop(self):
        '''flagcmd: unapply tfcrop agent'''
        # Remove any cmd from table
        flagcmd(vis=self.vis, action='clear', clearall=True)

        # Flag using manual agent
        myinput = "scan='1'"
        filename = create_input(myinput)
        flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True)
        
        # Flag using tfcrop agent from file
        # Note : For this test, scan=4 gives identical flags on 32/64 bit machines,
        #           and one flag difference on a Mac (32)
        #           Other scans give differences at the 0.005% level.
        myinput = "scan='4' mode=tfcrop correlation='ABS_RR' extendflags=False"
        filename = create_input(myinput)
        flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True,
                flagbackup=False)
        res = flagdata(vis=self.vis,mode='summary')
        self.assertEqual(res['scan']['1']['flagged'], 568134, 'Whole scan=1 should be flagged')
        #self.assertEqual(res['scan']['4']['flagged'], 1201, 'scan=4 should be partially flagged')
        self.assertTrue(res['scan']['4']['flagged']>= 1200 and res['scan']['4']['flagged']<= 1204, \
                        'scan=4 should be partially flagged')
        
        # Unapply only the tfcrop line
        flagcmd(vis=self.vis, action='unapply', useapplied=True, tablerows=1, savepars=False)
        result = flagdata(vis=self.vis,mode='summary',scan='4')
        self.assertEqual(result['flagged'], 0, 'Expected 0 flags, found %s'%result['flagged'])
        self.assertEqual(result['total'], 95256,'Expected total 95256, found %s'%result['total'])

 def test_save_cal(self):
     '''Flagcmd: list flag cmds from MS and save to a file'''
     self.setUp_data4rflag()
     msfile = self.vis
     flagcmd(vis=msfile, action='clear', clearall=True)
     
     # Save cmds to FLAG_CMD table
     flagcmd(vis=msfile, inpmode='list', inpfile=["spw='1,3",
                                                  "mode='clip' clipminmax=[0,500] datacolumn='SNR'"], 
             action='list', savepars=True)
     
     self.setUp_bpass_case()
     flagcmds = 'calflags.txt'
     if os.path.exists(flagcmds):
         os.system('rm -rf '+flagcmds)
         
     # Apply to cal table and save to an external file
     flagcmd(vis=self.vis, inpmode='table', inpfile=msfile, savepars=True, outfile=flagcmds)
     self.assertTrue(os.path.exists(flagcmds))
     res = flagdata(vis=self.vis, mode='summary')
     self.assertEqual(res['flagged'], 246315)
     
     # Apply from list in file and compare
     flagdata(vis=self.vis, mode='unflag')
     flagcmd(vis=self.vis, inpmode='list', inpfile=flagcmds, action='apply')
     res = flagdata(vis=self.vis, mode='summary')
     self.assertEqual(res['flagged'], 246315)

    def test_caltable_flagbackup(self):
        '''Flagmanager:: cal table mode=list, flagbackup=True/False'''
        # Need a fresh start
        os.system('rm -rf cal.fewscans.bpass*')
        self.setUp_bpass_case()
        self.unflag_ms()        
       
        flagmanager(vis=self.vis, mode='list')
        aflocal.open(self.vis)
        self.assertEqual(len(aflocal.getflagversionlist()), 2)
        aflocal.done()

        flagdata(vis=self.vis, mode='unflag', flagbackup=False)
        flagmanager(vis=self.vis, mode='list')
        aflocal.open(self.vis)
        self.assertEqual(len(aflocal.getflagversionlist()), 2)
        aflocal.done()

        flagdata(vis=self.vis, mode='unflag', flagbackup=True)
        flagmanager(vis=self.vis, mode='list')
        aflocal.open(self.vis)
        self.assertEqual(len(aflocal.getflagversionlist()), 3)
        aflocal.done()
        
        newname = 'Ha! The best version ever!'

        flagmanager(vis=self.vis, mode='rename', oldname='flagdata_1', versionname=newname, 
                    comment='This is a *much* better name')
        flagmanager(vis=self.vis, mode='list')
        aflocal.open(self.vis)
        self.assertEqual(len(aflocal.getflagversionlist()), 3)
        aflocal.done()
        
        self.assertTrue(os.path.exists(self.vis+'.flagversions/flags.'+newname),
                        'Flagversion file does not exist: flags.'+newname)        

    def test_uquack(self):
        '''flagcmd: unapply quack agent'''
        # Remove any cmd from table
        flagcmd(vis=self.vis, action='clear', clearall=True)

        # Flag using the quack agent
        myinput = "scan='1~3' mode=quack quackinterval=1.0"
        filename = create_input(myinput)
        flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True,
                flagbackup=False)
        result = flagdata(vis=self.vis,mode='summary')
        quack_flags = result['scan']['1']['flagged']

        # Flag using manual agent
        myinput = "scan='1'"
        filename = create_input(myinput)
        flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True,
                flagbackup=False)
        result = flagdata(vis=self.vis,mode='summary')
        scan1_flags = result['scan']['1']['flagged']

        # Unapply only the quack line
        flagcmd(vis=self.vis, action='unapply', useapplied=True, tablerows=0, savepars=True)
        result = flagdata(vis=self.vis,mode='summary')
        manual_flags = result['scan']['1']['flagged']
        
        # Only the manual flags should be there
#         self.assertEqual(result['flagged'], 568134, 'Expected 568134 flags, found %s'%result['flagged'])
#         self.assertEqual(result['total'], 568134,'Expected total 568134, found %s'%result['total'])
        # CAS-5377. New unapply action
        self.assertEqual(result['scan']['3']['flagged'], 0)
        self.assertEqual(manual_flags,scan1_flags - quack_flags)

    def test_savepars(self):
        '''importevla: save the flag commands and do not apply'''
        msname = 'notapplied.ms'
        cmdfile = msname.replace('.ms','_cmd.txt')
        if os.path.exists(msname):
            os.system('rm -rf '+msname)
        if os.path.exists(cmdfile):
            os.system('rm -rf '+cmdfile)

        importevla(asdm=self.asdm, vis=msname,scans='11~13',online=True,flagzero=True,shadow=True,savecmds=True,
                    applyflags=False,flagbackup=False)
        
        # Check flags
        res = flagdata(vis=msname, mode='summary')
        self.assertEqual(res['flagged'],0,'No flags should have been applied')

        # Check output file existence
        self.assertTrue(os.path.exists(cmdfile))
        
        # Check file content
        ff = open(cmdfile,'r')
        cmdlist = ff.readlines()
        ncmds = cmdlist.__len__()
        ff.close()
        self.assertEqual(ncmds, 216, 'Online, shadow and clip zeros should be saved to file')
        
        # Apply flags using flagdata
        flagdata(vis=msname, mode='list', inpfile=cmdfile)
        
        res = flagdata(vis=msname, mode='summary')
        self.assertEqual(res['flagged'],6090624)

    def test2m(self):
        """flagmanager test2m: Create, then restore autoflag"""

        self.unflag_ms()        

        flagdata(vis=self.vis, mode='summary')
        flagmanager(vis=self.vis)
        
        flagdata(vis=self.vis, mode='manual', antenna="2", flagbackup=True)
        
        flagmanager(vis=self.vis)
        ant2 = flagdata(vis=self.vis, mode='summary')['flagged']

        print "After flagging antenna 2 there were", ant2, "flags"

        # Change flags, then restore
        flagdata(vis=self.vis, mode='manual', antenna="3", flagbackup=True)
        flagmanager(vis = self.vis)
        ant3 = flagdata(vis=self.vis, mode='summary')['flagged']

        print "After flagging antenna 2 and 3 there were", ant3, "flags"

        flagmanager(vis=self.vis, mode='restore', versionname='flagdata_2')
        restore2 = flagdata(vis=self.vis, mode='summary')['flagged']

        print "After restoring pre-antenna 2 flagging, there are", restore2, "flags; should be", ant2

        self.assertEqual(restore2, ant2)

 def test_MS_flagcmds(self):
     """Flagcmd:: Save flags to MS and apply to cal table"""
     self.setUp_data4rflag()
     msfile = self.vis
     flagcmd(vis=msfile, action='clear', clearall=True)
     flagdata(vis=msfile, antenna='ea09', action='', savepars=True)
     
     self.setUp_bpass_case()
     flagcmd(vis=self.vis, inpfile=msfile, action='apply', flagbackup=False)
     summary=flagdata(vis=self.vis, mode='summary')
     self.assertEqual(summary['antenna']['ea09']['flagged'], 48000.0)
     self.assertEqual(summary['antenna']['ea10']['flagged'], 0.0)

    def test_apply4(self):
        """importevla: Save online flags to FLAG_CMD and file; do not apply"""

        # Use default msname and outfile
        msname = "xosro4.ms"
        cmdfile = msname.replace(".ms", "_cmd.txt")
        if os.path.exists(msname):
            os.system("rm -rf " + msname)
        if os.path.exists(cmdfile):
            os.system("rm -rf " + cmdfile)

        importevla(
            asdm=self.asdm,
            vis=msname,
            scans="2",
            online=True,
            shadow=False,
            flagzero=False,
            applyflags=False,
            savecmds=True,
            flagbackup=False,
        )

        # No flags were applied
        res = flagdata(vis=msname, mode="summary")
        self.assertEqual(res["flagged"], 0)

        # Apply only row 213 using flagcmd
        # The command in row 213 is the following:
        # antenna='ea06' timerange='2012/02/22/22:30:55.200~2012/02/22/22:35:08.199'
        # spw='EVLA_X#A0C0#0' correlation='LL,LR,RL
        flagcmd(vis=msname, action="apply", tablerows=213)

        # Check flags. RR should no be flagged
        res = flagdata(vis=msname, mode="summary")
        self.assertEqual(res["correlation"]["RR"]["flagged"], 0, "RR should not be flagged")
        self.assertEqual(res["correlation"]["LL"]["flagged"], 29440)
        self.assertEqual(res["correlation"]["LR"]["flagged"], 29440)
        self.assertEqual(res["correlation"]["RL"]["flagged"], 29440)
        self.assertEqual(res["antenna"]["ea06"]["flagged"], 88320)
        self.assertEqual(res["antenna"]["ea07"]["flagged"], 3840, "Only a few baselines should be flagged")
        self.assertEqual(res["antenna"]["ea08"]["flagged"], 3840, "Only a few baselines should be flagged")

        # Check output file existence
        self.assertTrue(os.path.exists(cmdfile))

        # Check file content
        ff = open(cmdfile, "r")
        cmdlist = ff.readlines()
        ncmds = cmdlist.__len__()
        ff.close()
        self.assertEqual(ncmds, 214, "Only Online cmds should have been saved to file")

    def test_flagsrestore(self):
        """Partition: check that we can restore the flags"""
        # Delete any flagversions
        if os.path.exists(self.msfile + ".flagversions"):
            shutil.rmtree(self.msfile + ".flagversions")

        # Unflag the MS
        flagdata(vis=self.msfile, mode="unflag", flagbackup=False)

        # Run partition and create the .flagversions
        partition(vis=self.msfile, outputvis=self.mmsfile, createmms=True, disableparallel=True, datacolumn="data")
        self.assertTrue(os.path.exists(self.mmsfile + ".flagversions"))

        # Flag spw=9 and then spw=7 in the MMS
        flagdata(vis=self.mmsfile, mode="manual", spw="9", flagbackup=True)
        flagdata(vis=self.mmsfile, mode="manual", spw="7", flagbackup=True)

        # There should be flags in spw=7 and 9 in MMS
        res = flagdata(vis=self.mmsfile, mode="summary")
        self.assertEqual(res["flagged"], 549888)

        # Restore the original flags (there should be none)
        flagmanager(vis=self.mmsfile, mode="restore", versionname="partition_1")
        res = flagdata(vis=self.mmsfile, mode="summary")
        self.assertEqual(res["flagged"], 0)

 def test_flagsrestore(self):
     '''Partition: check that we can restore the flags'''
     # Delete any flagversions
     if os.path.exists(self.msfile+'.flagversions'):
         shutil.rmtree(self.msfile+'.flagversions')
         
     # Unflag the MS
     flagdata(vis=self.msfile, mode='unflag', flagbackup=False)
     
     # Run partition and create the .flagversions
     partition(vis=self.msfile, outputvis=self.mmsfile, createmms=True,
               disableparallel=True, datacolumn='data')
     self.assertTrue(os.path.exists(self.mmsfile+'.flagversions'))
     
     # Flag spw=9 and then spw=7 in the MMS
     flagdata(vis=self.mmsfile, mode='manual', spw='9', flagbackup=True)
     flagdata(vis=self.mmsfile, mode='manual', spw='7', flagbackup=True)
     
     # There should be flags in spw=7 and 9 in MMS
     res = flagdata(vis=self.mmsfile, mode='summary')
     self.assertEqual(res['flagged'],549888)
     
     # Restore the original flags (there should be none)
     flagmanager(vis=self.mmsfile, mode='restore', versionname='partition_1')
     res = flagdata(vis=self.mmsfile, mode='summary')
     self.assertEqual(res['flagged'],0)

 def test_clip_one_list(self):
     '''Flagcmd: Flag one solution using one command in a list'''
     flist = ["mode='clip' clipminmax=[0,3] correlation='REAL_Sol1' datacolumn='CPARAM'"]
     flagcmd(vis=self.vis, inpmode='list', inpfile=flist)
     res = flagdata(vis=self.vis, mode='summary')
     self.assertEqual(res['flagged'], 309388)
     self.assertEqual(res['correlation']['Sol2']['flagged'], 0)

 def test_default_cparam(self):
     '''Flagcmd: flag CPARAM data column'''
     flist = ["mode='clip' clipzeros=True datacolumn='CPARAM'"]
     flagcmd(vis=self.vis, inpmode='list', inpfile=flist,
             flagbackup=False)
     res = flagdata(vis=self.vis, mode='summary')
     self.assertEqual(res['flagged'], 11078, 'Should use CPARAM as the default column')

 def test_unsupported_mode_in_list(self):
     '''Flagcmd: elevation and shadow are not supported in cal tables'''
     res = flagcmd(vis=self.vis, inpmode='list', inpfile=["mode='elevation'",
                                                          "spw='1'"])
     
     res = flagdata(vis=self.vis, mode='summary')
     self.assertEqual(res['spw']['1']['flagged'], 83200)

    def test_apply3(self):
        '''importevla: apply clip zeros on RR and LL and save to file'''
        msname = 'zeros.ms'
        cmdfile = 'zeros_cmd.txt'
        if os.path.exists(msname):
            os.system('rm -rf '+msname)
        if os.path.exists(cmdfile):
            os.system('rm -rf '+cmdfile)
            
            
        importevla(asdm=self.asdm, vis=msname, scans='2,13',online=False, applyflags=True,
                    shadow=False,flagzero=True,flagpol=False, flagbackup=False, savecmds=True)
        
        # Check flags
        res = flagdata(vis=msname, mode='summary')
        self.assertEqual(res['flagged'],0,'There are no zeros in this data set')
        self.assertEqual(res['scan']['2']['flagged'],0,'No flags should have been applied')
        self.assertEqual(res['scan']['13']['flagged'],0,'No flags should have been applied')
        
        # Check output file existence
        self.assertTrue(os.path.exists(cmdfile))

        # Check output file existence
        self.assertTrue(os.path.exists(cmdfile))
        
        # Check file content
        ff = open(cmdfile,'r')
        cmdlist = ff.readlines()
        ncmds = cmdlist.__len__()
        self.assertEqual(ncmds, 2, 'Only clip zeros should be saved to file')

    def test_vla_mixed_polarizations_mms2(self):
        
        self.outputms = 'test_vla_mixed_polarizations_2.mms'
        
        mstransform(vis=self.vis,outputvis=self.outputms,scan='16',datacolumn='DATA', createmms=True,
                    separationaxis='spw',spw='16~18',correlation='XX')
        
        # Check that DDI sub-table is consistent with POLARIZATION sub-table
        mytb = tbtool()
        mytb.open(self.outputms + '/POLARIZATION')
        npols = mytb.nrows()
        mytb.close()
        
        mytb = tbtool()
        mytb.open(self.outputms + '/DATA_DESCRIPTION')
        polIds = mytb.getcol('POLARIZATION_ID')
        mytb.close()    
        for id in polIds:
            self.assertTrue(id in range(npols),'PolarizationId in DATA_DESCRIPTION not consistent with POLARIZATION table')
        
#        self.assertTrue(all(polIds < npols), 'PolarizationId in DATA_DESCRIPTION not consistent with POLARIZATION table') 
        
        # Check that flagdata can run properly with output MS
        summary = flagdata(vis=self.outputms,mode='summary')
        self.assertTrue(summary.has_key('correlation'), 'Flagdata failure due to missformated MS') 

 def test_compatibility(self):
     myinput = "observation='1' mode='manualflag'"
     filename = create_input(myinput)
     
     flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=False,
             flagbackup=False)
     test_eq(flagdata(vis=self.vis, mode='summary'), 2882778, 28500)

    def test_apply1(self):
        """importevla: apply all flags and save to file"""
        msname = "online.ms"
        cmdfile = "online_cmd.txt"
        if os.path.exists(msname):
            os.system("rm -rf " + msname)
        if os.path.exists(cmdfile):
            os.system("rm -rf " + cmdfile)

        importevla(
            asdm=self.asdm,
            vis=msname,
            scans="2",
            online=True,
            applyflags=True,
            shadow=True,
            flagzero=True,
            flagbackup=False,
            savecmds=True,
        )

        # Check flags
        res = flagdata(vis=msname, mode="summary")
        self.assertEqual(res["flagged"], 2446080)

        # Check output file existence
        self.assertTrue(os.path.exists(cmdfile))

        # Check file content
        ff = open(cmdfile, "r")
        cmdlist = ff.readlines()
        ncmds = cmdlist.__len__()
        self.assertEqual(ncmds, 216)

 def test_CAS4819(self):
     '''flagcmd: CAS-4819, Flag commands from three files'''
     self.setUp_ngc5921()
     # creat first input file
     myinput = "scan='1'\n"\
             "scan='2'\n"\
             "scan='3'"
     filename1 = 'list7a.txt'
     create_input1(myinput, filename1)
     
     # Create second input file
     myinput = "scan='5'\n"\
             "scan='6'\n"\
             "scan='7'"        
     filename2 = 'list7b.txt'
     create_input1(myinput, filename2)
     
      # Create third input file
     myinput = "scan='4' mode='clip' clipminmax=[0,4]" 
     filename3 = 'list7c.txt'
     create_input1(myinput, filename3)
    
     flagcmd(vis=self.vis, inpmode='list', inpfile=[filename1,filename2,filename3],
              flagbackup=False, action='apply')
     
     res = flagdata(vis=self.vis, mode='summary')
     self.assertEqual(res['scan']['1']['flagged'], 568134)
     self.assertEqual(res['scan']['2']['flagged'], 238140)
     self.assertEqual(res['scan']['3']['flagged'], 762048)
     self.assertEqual(res['scan']['4']['flagged'], 6696)
     self.assertEqual(res['scan']['5']['flagged'], 142884)
     self.assertEqual(res['scan']['6']['flagged'], 857304)
     self.assertEqual(res['scan']['7']['flagged'], 190512)
     self.assertEqual(res['total'],2854278)
     self.assertEqual(res['flagged'],2765718)

 def test_list_reason1(self):
     '''flagcmd: select by reason from two files'''
     self.setUp_ngc5921()
     # creat first input file
     myinput = "scan='1' spw='0:10~20' reason='NONE'\n"\
             "scan='2' reason='EVEN'\n"\
             "scan='3' reason='ODD'"
     filename1 = 'reasonfile1.txt'
     create_input1(myinput, filename1)
     
     # Create second input file
     myinput = "scan='5' reason='ODD'\n"\
             "scan='6' reason='EVEN'\n"\
             "scan='7' reason='ODD'"        
     filename2 = 'reasonfile2.txt'
     create_input1(myinput, filename2)
     
     # Apply flag cmds on ODD reason
     flagcmd(vis=self.vis, inpmode='list', inpfile=[filename1,filename2], reason='ODD',
              flagbackup=False, action='apply')
     
     res = flagdata(vis=self.vis, mode='summary')
     self.assertEqual(res['scan']['3']['flagged'], 762048)
     self.assertEqual(res['scan']['5']['flagged'], 142884)
     self.assertEqual(res['scan']['6']['flagged'], 0)
     self.assertEqual(res['scan']['7']['flagged'], 190512)
     self.assertEqual(res['flagged'], 762048+142884+190512)

    def test_umanualflag(self):
        '''flagcmd: unapply manual agent'''
        # Remove any cmd from table
        flagcmd(vis=self.vis, action='clear', clearall=True)

        # Flag using manual agent
        myinput = "scan='1'"
        filename = create_input(myinput)
        flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True,
                flagbackup=False)

        # Flag using the quack agent
        myinput = "scan='1~3' mode=quack quackinterval=1.0"
        filename = create_input(myinput)
        flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True,
                flagbackup=False)
        
        # Unapply the manual line
        flagcmd(vis=self.vis, action='unapply', useapplied=True, tablerows=0, savepars=False)
        result = flagdata(vis=self.vis,mode='summary',scan='1,2,3')
        
        # scan 1 should be fully unflagged
        self.assertEqual(result['scan']['1']['flagged'], 0)
        self.assertEqual(result['scan']['2']['flagged'], 47628)
        self.assertEqual(result['scan']['3']['flagged'], 47628)
        self.assertEqual(result['flagged'], 47628+47628)