def test_default_cparam(self):
     '''Flagcmd: flag CPARAM data column'''
     flist = ["mode='clip' clipzeros=True datacolumn='CPARAM'"]
     flagcmd(vis=self.vis, inpmode='list', inpfile=flist,
             flagbackup=False)
     res = flagdata(vis=self.vis, mode='summary')
     self.assertEqual(res['flagged'], 11078, 'Should use CPARAM as the default column')

 def test_compatibility(self):
     myinput = "observation='1' mode='manualflag'"
     filename = create_input(myinput)
     
     flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=False,
             flagbackup=False)
     test_eq(flagdata(vis=self.vis, mode='summary'), 2882778, 28500)

 def test_clip_one_list(self):
     '''Flagcmd: Flag one solution using one command in a list'''
     flist = ["mode='clip' clipminmax=[0,3] correlation='REAL_Sol1' datacolumn='CPARAM'"]
     flagcmd(vis=self.vis, inpmode='list', inpfile=flist)
     res = flagdata(vis=self.vis, mode='summary')
     self.assertEqual(res['flagged'], 309388)
     self.assertEqual(res['correlation']['Sol2']['flagged'], 0)

 def test_CAS4819(self):
     '''flagcmd: CAS-4819, Flag commands from three files'''
     self.setUp_ngc5921()
     # creat first input file
     myinput = "scan='1'\n"\
             "scan='2'\n"\
             "scan='3'"
     filename1 = 'list7a.txt'
     create_input1(myinput, filename1)
     
     # Create second input file
     myinput = "scan='5'\n"\
             "scan='6'\n"\
             "scan='7'"        
     filename2 = 'list7b.txt'
     create_input1(myinput, filename2)
     
      # Create third input file
     myinput = "scan='4' mode='clip' clipminmax=[0,4]" 
     filename3 = 'list7c.txt'
     create_input1(myinput, filename3)
    
     flagcmd(vis=self.vis, inpmode='list', inpfile=[filename1,filename2,filename3],
              flagbackup=False, action='apply')
     
     res = flagdata(vis=self.vis, mode='summary')
     self.assertEqual(res['scan']['1']['flagged'], 568134)
     self.assertEqual(res['scan']['2']['flagged'], 238140)
     self.assertEqual(res['scan']['3']['flagged'], 762048)
     self.assertEqual(res['scan']['4']['flagged'], 6696)
     self.assertEqual(res['scan']['5']['flagged'], 142884)
     self.assertEqual(res['scan']['6']['flagged'], 857304)
     self.assertEqual(res['scan']['7']['flagged'], 190512)
     self.assertEqual(res['total'],2854278)
     self.assertEqual(res['flagged'],2765718)

 def test_list_reason1(self):
     '''flagcmd: select by reason from two files'''
     self.setUp_ngc5921()
     # creat first input file
     myinput = "scan='1' spw='0:10~20' reason='NONE'\n"\
             "scan='2' reason='EVEN'\n"\
             "scan='3' reason='ODD'"
     filename1 = 'reasonfile1.txt'
     create_input1(myinput, filename1)
     
     # Create second input file
     myinput = "scan='5' reason='ODD'\n"\
             "scan='6' reason='EVEN'\n"\
             "scan='7' reason='ODD'"        
     filename2 = 'reasonfile2.txt'
     create_input1(myinput, filename2)
     
     # Apply flag cmds on ODD reason
     flagcmd(vis=self.vis, inpmode='list', inpfile=[filename1,filename2], reason='ODD',
              flagbackup=False, action='apply')
     
     res = flagdata(vis=self.vis, mode='summary')
     self.assertEqual(res['scan']['3']['flagged'], 762048)
     self.assertEqual(res['scan']['5']['flagged'], 142884)
     self.assertEqual(res['scan']['6']['flagged'], 0)
     self.assertEqual(res['scan']['7']['flagged'], 190512)
     self.assertEqual(res['flagged'], 762048+142884+190512)

 def test_cal_observation(self):
     '''Flagcmd: flag an observation from an old cal table format'''
     # Note: this cal table does not have an observation column. 
     # The column and sub-table should be added and the flagging
     # should happen after this.
     flagcmd(vis=self.vis, inpmode='list', inpfile=["observation='0'"], flagbackup=False)
     res=flagdata(vis=self.vis, mode='summary')
     self.assertEqual(res['flagged'],1248000)
     self.assertEqual(res['total'],1248000)

 def test_xml2(self):
     '''flagcmd: list xml file and save in outfile'''
     
     # The MS only contains clip and shadow commands
     
     # Apply the shadow command
     flagcmd(vis=self.vis, action='apply', reason='SHADOW', flagbackup=False)
     res = flagdata(vis=self.vis, mode='summary')
     self.assertEqual(res['flagged'], 240640)

 def test_manual_field_selection_for_bpass(self):
     """Flagcmd:: Manually flag a bpass-based CalTable using field selection"""
     
     flagcmd(vis=self.vis, inpmode='list', inpfile=["field='3C286_A'"], flagbackup=False)
     summary=flagdata(vis=self.vis, mode='summary')
     
     self.assertEqual(summary['field']['3C286_A']['flagged'], 499200.0)
     self.assertEqual(summary['field']['3C286_B']['flagged'], 0)
     self.assertEqual(summary['field']['3C286_C']['flagged'], 0)
     self.assertEqual(summary['field']['3C286_D']['flagged'], 0)

 def test_action_plot_table(self):
     '''flagcmd: Test action=plot, nothing plotted'''
     outplot = 'noplot.png'
     flagcmd(vis=self.vis, inpmode='list', 
         inpfile=["intent='CAL*POINT*' field=''","scan='5'"], 
         action='list', savepars=True)
     
     flagcmd(vis=self.vis, inpmode='table', useapplied=True, action='plot',
             plotfile=outplot)
     
     self.assertTrue(os.path.exists(outplot),'Plot file was not created')

 def test_action_plot_list(self):
     '''flagcmd: Test action=plot to plot 4 antennas and no timerange'''
     outplot = 'fourplot.png'
     cmds = ["antenna='ea01' reason='none'",
             "antenna='ea11' reason='no_reason'",
             "antenna='ea19' reason='none'",
             "antenna='ea24' reason='other'"]
     
     flagcmd(vis=self.vis, inpmode='list', inpfile=cmds, action='plot',plotfile=outplot)
             
     self.assertTrue(os.path.exists(outplot),'Plot file was not created')

 def test_MS_flagcmds(self):
     """Flagcmd:: Save flags to MS and apply to cal table"""
     self.setUp_data4rflag()
     msfile = self.vis
     flagcmd(vis=msfile, action='clear', clearall=True)
     flagdata(vis=msfile, antenna='ea09', action='', savepars=True)
     
     self.setUp_bpass_case()
     flagcmd(vis=self.vis, inpfile=msfile, action='apply', flagbackup=False)
     summary=flagdata(vis=self.vis, mode='summary')
     self.assertEqual(summary['antenna']['ea09']['flagged'], 48000.0)
     self.assertEqual(summary['antenna']['ea10']['flagged'], 0.0)

    def test_list_field_Selection_for_bpass(self):
        """Flagcmd:: Manually flag a bpass-based CalTable using file in list mode """
        
        myinput = "field='3C286_A'"
        filename = create_input(myinput)

        flagcmd(vis=self.vis, inpmode='list', inpfile=filename, flagbackup=False)
        summary=flagdata(vis=self.vis, mode='summary')
        self.assertEqual(summary['field']['3C286_A']['flagged'], 499200.0)
        self.assertEqual(summary['field']['3C286_B']['flagged'], 0)
        self.assertEqual(summary['field']['3C286_C']['flagged'], 0)
        self.assertEqual(summary['field']['3C286_D']['flagged'], 0)

 def test_cmd(self):
     '''flagcmd: inpmode=list with empty parameter'''
     
     # Test the correct parsing with empty parameter such as antenna=''
     flagcmd(vis=self.vis, inpmode='list', 
              inpfile=["intent='CAL*POINT*' field=''","scan='1,8' antenna=''","scan='10'"], 
              action='apply', savepars=False, flagbackup=False)
     res = flagdata(vis=self.vis,mode='summary')
     self.assertEqual(res['scan']['1']['flagged'], 192416)
     self.assertEqual(res['scan']['8']['flagged'], 39096)
     self.assertEqual(res['scan']['10']['flagged'], 39096)
     self.assertEqual(res['flagged'], 270608)

    def test_apply4(self):
        """importevla: Save online flags to FLAG_CMD and file; do not apply"""

        # Use default msname and outfile
        msname = "xosro4.ms"
        cmdfile = msname.replace(".ms", "_cmd.txt")
        if os.path.exists(msname):
            os.system("rm -rf " + msname)
        if os.path.exists(cmdfile):
            os.system("rm -rf " + cmdfile)

        importevla(
            asdm=self.asdm,
            vis=msname,
            scans="2",
            online=True,
            shadow=False,
            flagzero=False,
            applyflags=False,
            savecmds=True,
            flagbackup=False,
        )

        # No flags were applied
        res = flagdata(vis=msname, mode="summary")
        self.assertEqual(res["flagged"], 0)

        # Apply only row 213 using flagcmd
        # The command in row 213 is the following:
        # antenna='ea06' timerange='2012/02/22/22:30:55.200~2012/02/22/22:35:08.199'
        # spw='EVLA_X#A0C0#0' correlation='LL,LR,RL
        flagcmd(vis=msname, action="apply", tablerows=213)

        # Check flags. RR should no be flagged
        res = flagdata(vis=msname, mode="summary")
        self.assertEqual(res["correlation"]["RR"]["flagged"], 0, "RR should not be flagged")
        self.assertEqual(res["correlation"]["LL"]["flagged"], 29440)
        self.assertEqual(res["correlation"]["LR"]["flagged"], 29440)
        self.assertEqual(res["correlation"]["RL"]["flagged"], 29440)
        self.assertEqual(res["antenna"]["ea06"]["flagged"], 88320)
        self.assertEqual(res["antenna"]["ea07"]["flagged"], 3840, "Only a few baselines should be flagged")
        self.assertEqual(res["antenna"]["ea08"]["flagged"], 3840, "Only a few baselines should be flagged")

        # Check output file existence
        self.assertTrue(os.path.exists(cmdfile))

        # Check file content
        ff = open(cmdfile, "r")
        cmdlist = ff.readlines()
        ncmds = cmdlist.__len__()
        ff.close()
        self.assertEqual(ncmds, 214, "Only Online cmds should have been saved to file")

    def test_utfcrop(self):
        '''flagcmd: unapply tfcrop agent'''
        # Remove any cmd from table
        flagcmd(vis=self.vis, action='clear', clearall=True)

        # Flag using manual agent
        myinput = "scan='1'"
        filename = create_input(myinput)
        flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True)
        
        # Flag using tfcrop agent from file
        # Note : For this test, scan=4 gives identical flags on 32/64 bit machines,
        #           and one flag difference on a Mac (32)
        #           Other scans give differences at the 0.005% level.
        myinput = "scan='4' mode=tfcrop correlation='ABS_RR' extendflags=False"
        filename = create_input(myinput)
        flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True,
                flagbackup=False)
        res = flagdata(vis=self.vis,mode='summary')
        self.assertEqual(res['scan']['1']['flagged'], 568134, 'Whole scan=1 should be flagged')
        #self.assertEqual(res['scan']['4']['flagged'], 1201, 'scan=4 should be partially flagged')
        self.assertTrue(res['scan']['4']['flagged']>= 1200 and res['scan']['4']['flagged']<= 1204, \
                        'scan=4 should be partially flagged')
        
        # Unapply only the tfcrop line
        flagcmd(vis=self.vis, action='unapply', useapplied=True, tablerows=1, savepars=False)
        result = flagdata(vis=self.vis,mode='summary',scan='4')
        self.assertEqual(result['flagged'], 0, 'Expected 0 flags, found %s'%result['flagged'])
        self.assertEqual(result['total'], 95256,'Expected total 95256, found %s'%result['total'])

 def test_save_cal(self):
     '''Flagcmd: list flag cmds from MS and save to a file'''
     self.setUp_data4rflag()
     msfile = self.vis
     flagcmd(vis=msfile, action='clear', clearall=True)
     
     # Save cmds to FLAG_CMD table
     flagcmd(vis=msfile, inpmode='list', inpfile=["spw='1,3",
                                                  "mode='clip' clipminmax=[0,500] datacolumn='SNR'"], 
             action='list', savepars=True)
     
     self.setUp_bpass_case()
     flagcmds = 'calflags.txt'
     if os.path.exists(flagcmds):
         os.system('rm -rf '+flagcmds)
         
     # Apply to cal table and save to an external file
     flagcmd(vis=self.vis, inpmode='table', inpfile=msfile, savepars=True, outfile=flagcmds)
     self.assertTrue(os.path.exists(flagcmds))
     res = flagdata(vis=self.vis, mode='summary')
     self.assertEqual(res['flagged'], 246315)
     
     # Apply from list in file and compare
     flagdata(vis=self.vis, mode='unflag')
     flagcmd(vis=self.vis, inpmode='list', inpfile=flagcmds, action='apply')
     res = flagdata(vis=self.vis, mode='summary')
     self.assertEqual(res['flagged'], 246315)

    def test_umanualflag(self):
        '''flagcmd: unapply manual agent'''
        # Remove any cmd from table
        flagcmd(vis=self.vis, action='clear', clearall=True)

        # Flag using manual agent
        myinput = "scan='1'"
        filename = create_input(myinput)
        flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True,
                flagbackup=False)

        # Flag using the quack agent
        myinput = "scan='1~3' mode=quack quackinterval=1.0"
        filename = create_input(myinput)
        flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True,
                flagbackup=False)
        
        # Unapply the manual line
        flagcmd(vis=self.vis, action='unapply', useapplied=True, tablerows=0, savepars=False)
        result = flagdata(vis=self.vis,mode='summary',scan='1,2,3')
        
        # scan 1 should be fully unflagged
        self.assertEqual(result['scan']['1']['flagged'], 0)
        self.assertEqual(result['scan']['2']['flagged'], 47628)
        self.assertEqual(result['scan']['3']['flagged'], 47628)
        self.assertEqual(result['flagged'], 47628+47628)

 def test_cal_time_field(self):
     '''Flagcmd: clip a timerange from a field'''
     # this timerange corresponds to field 3C286_D
     flags = "mode='clip' timerange='>14:58:33.6' clipzeros=True clipminmax=[0.,0.4]"\
             " datacolumn='CPARAM'"
     
     # Apply the flags
     flagcmd(vis=self.vis, inpmode='list', inpfile=[flags], flagbackup=False)
     res=flagdata(vis=self.vis, mode='summary')
     self.assertEqual(res['field']['3C286_A']['flagged'],0)
     self.assertEqual(res['field']['3C286_B']['flagged'],0)
     self.assertEqual(res['field']['3C286_C']['flagged'],0)
     self.assertEqual(res['field']['3C286_D']['flagged'],2221)
     self.assertEqual(res['flagged'],2221)

    def test_uquack(self):
        '''flagcmd: unapply quack agent'''
        # Remove any cmd from table
        flagcmd(vis=self.vis, action='clear', clearall=True)

        # Flag using the quack agent
        myinput = "scan='1~3' mode=quack quackinterval=1.0"
        filename = create_input(myinput)
        flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True,
                flagbackup=False)
        result = flagdata(vis=self.vis,mode='summary')
        quack_flags = result['scan']['1']['flagged']

        # Flag using manual agent
        myinput = "scan='1'"
        filename = create_input(myinput)
        flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True,
                flagbackup=False)
        result = flagdata(vis=self.vis,mode='summary')
        scan1_flags = result['scan']['1']['flagged']

        # Unapply only the quack line
        flagcmd(vis=self.vis, action='unapply', useapplied=True, tablerows=0, savepars=True)
        result = flagdata(vis=self.vis,mode='summary')
        manual_flags = result['scan']['1']['flagged']
        
        # Only the manual flags should be there
#         self.assertEqual(result['flagged'], 568134, 'Expected 568134 flags, found %s'%result['flagged'])
#         self.assertEqual(result['total'], 568134,'Expected total 568134, found %s'%result['total'])
        # CAS-5377. New unapply action
        self.assertEqual(result['scan']['3']['flagged'], 0)
        self.assertEqual(manual_flags,scan1_flags - quack_flags)

    def test_intent(self):
        '''flagcmd: test scan intent selection'''
        
        myinput = "intent='CAL*POINT*'\n"\
                "#scan=3,4"
        filename = create_input(myinput)
        
        # flag POINTING CALIBRATION scans and ignore comment line
        flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=False,
                flagbackup=False)
#         test_eq(flagdata(vis=self.vis,mode='summary', antenna='2'), 377280, 26200)
        res = flagdata(vis=self.vis,mode='summary')
#         self.assertEqual(res['scan']['1']['flagged'], 80184, 'Only scan 1 should be flagged')
#         self.assertEqual(res['scan']['4']['flagged'], 0, 'Scan 4 should not be flagged')
        self.assertEqual(res['scan']['1']['flagged'], 192416.0)