def run_lephem(fname, time):
    """
    run idl script lephem.pro which convert the data into ascii data
    input:  fname   --- the name of the input file: DE<time>.EPH
            time    --- time stamp of the file copied
    output: ascii data file -- DE<time>.EPH.dat0
    """

    cmd = 'cp ' + fname + ' ' +  eph_data + '/.'
    os.system(cmd)

    cname = 'DE' + str(time) + '.EPH'
    
    try:
        line = "lephem,'" + str(cname) + "'\n"
        line = line + "exit\n"
        fo  = open('./eph_run.pro', 'w')
        fo.write(line)
        fo.close()

        os.environ['IDL_PATH'] = idl_path
        subprocess.call("idl eph_run.pro",  shell=True)
        mcf.rm_file('./eph_run.pro')
        return 1
    except:
        return 0

def mv_old_file(dom):

    dom -= 30
    if dom > 0:
        [year, ydate] = tcnv.DOMtoYdate(dom)
        sydate = str(ydate)
        if ydate < 10:
            sydate = '00' + sydate
        elif ydate < 100:
            sydate = '0' + sydate

        atime = str(year) + ':' + sydate + ':00:00:00'
        stime = tcnv.axTimeMTA(atime)

        cmd = 'ls ' + house_keeping + '/Defect/CCD*/* > ' + zspace
        os.system(cmd)
        fs = open(zspace, 'r')
        ldata = [line.strip() for line in fs.readlines()]
        fs.close()
        mcf.rm_file(zspace)
        for ent in ldata:
            atemp = re.split('\/acis', ent)
            btemp = re.split('_', atemp[1])
            if int(btemp[0]) < stime:
                out = ent
                out = out.replace('Defect', 'Defect/Save')
                cmd = 'mv ' + ent + ' ' + out 
                os.system(cmd)

def extract_cron_file_name():

    """
    extract cron error message file names for the current user/machine
    output: cron_file_name:   a list of cron file names (file names only no directory path)
    """

    try:
        cmd = 'crontab -l >' + zspace 
        os.system(cmd)
    
        f    = open(zspace, 'r')
        data = [line.strip() for line in f.readlines()]
        f.close()
        mcf.rm_file(zspace)
    except:
        exit(1)

    cron_file_name = []
    for ent in data:
        m = re.search('Logs', ent)
        if m is not None and ent[0] != '#':
            atemp = re.split('Logs/', ent)
            btemp = re.split('2>&1',  atemp[1])
            cron  = btemp[0]
            cron  = cron.strip()
            cron_file_name.append(cron)
#
#--- removing duplicated lines
#
    cron_file_name = list(set(cron_file_name))

    return cron_file_name

def update_html(update):
    """
    check whether the update is needed and if so, run the update
    input:  update  --- if it is 1, run the update without chekcing the file exist or not
    output: none, but updated html pages (in <web_dir>)
    """
#
#--- find today's date
#
    today = time.localtime()
    year  = today.tm_year
#
#--- if update is asked, just run the update
#
    if update > 0:
        run_update(year)
#
#--- otherwise, find the last update, and if needed, run the update
#
    else:
        cmd   = 'ls ' + web_dir +'*.html > ' + zspace
        os.system(cmd)
        f     = open(zspace, 'r')
        out   = f.read()
        f.close()
#
#--- chekcing the file existance (looking for year in the file name)
#
        mcf.rm_file(zspace)
        mc    = re.search(str(year), out)

        if mc is None:
            run_update(year)

def run_arc(inst, start, stop):
    """
    run arc4gl and extract evt2 data for "inst"
    input:  inst    --- instrument, acis or hrc
            start   --- interval start time in format of mm/dd/yy (e.g. 05/01/15)
            stop    --- interval stop time in format of mm/dd/yy
    """

    line = 'operation=retrieve\n'
    line = line + 'dataset=flight\n'
    line = line + 'detector=' + inst + '\n'
    line = line + 'level=2\n'
    line = line + 'filetype=evt2\n'
    line = line + 'tstart=' + start + '\n'
    line = line + 'tstop=' + stop  + '\n'
    line = line + 'go\n'
    f    = open(zspace, 'w')
    f.write(line)
    f.close()

    cmd1 = "/usr/bin/env PERL5LIB="
    cmd2 =  ' echo ' +  hakama + ' |arc4gl -U' + dare + ' -Sarcocc -i' + zspace
    cmd  = cmd1 + cmd2
#
#--- run arc4gl
#
    bash(cmd,  env=ascdsenv)
    mcf.rm_file(zspace)

def extract_data(start, stop):
    """
    extract data to compute HRMA focus plots
    input:  start   ---- start time in the foramt of mm/dd/yy (e.g. 05/01/15)
            stio    ---- stop time in the format of mm/dd/yy
    output: acis*evt2.fits.gz, hrc*evt2.fits.gz
    """
#
#--- check whether previous fits files are still around, and if so, remove them
#
    cmd = 'ls * > ' + zspace
    os.system(cmd)
    f   = open(zspace, 'r')
    chk = f.read(10000000)
    f.close()
    mcf.rm_file(zspace)
    mc  = re.search('fits', chk)
    if mc is not None:
        cmd = 'rm *fits*'
        os.system(cmd)
#
#--- if time interval is not given, set for a month interval
#
    if start == '':
        [start, stop] = set_interval()
#
#--- extract acis and hrc evt2 files
#
    inst = 'acis'
    run_arc(inst, start, stop)
    inst = 'hrc'
    run_arc(inst, start, stop)

def run_arc4gl(start, stop, operation='retrieve', dataset='flight', detector='telem', level='raw'):
    """
    extract data from archive using arc4gl
    input:  start   --- starting time in the format of mm/dd/yy,hh/mm/ss. hh/mm/ss is optional
            stop    --- stoping time
            operation   --- operation command.  default = retrieve
            dataset     --- dataset name.       default = flight
            detector    --- detector name       default = telem
            level       --- level               defalut = raw
    output: extracted data set
    """
#
#--- write arc4gl command
#
    line = 'operation = '       + operation  + '\n'
    line = line + 'dataset = '  + dataset    + '\n'
    line = line + 'detector = ' + detector   + '\n'
    line = line + 'level = '    + level      + '\n'
    line = line + 'tstart='     + str(start) + '\n'
    line = line + 'tstop='      + str(stop)  + '\n'
    line = line + 'go\n'

    fo = open(zspace, 'w')
    fo.write(line)
    fo.close()
#
#--- run arc4gl 
#
    cmd1 = '/usr/bin/env PERL5LIB=""'
    cmd2 = ' echo ' + hakama + '|arc4gl -U' + dare + ' -Sarcocc -i' + zspace
    cmd  = cmd1 + cmd2
    bash(cmd, env=ascdsenv)
    mcf.rm_file(zspace)

def get_dump_em_files(start, stop):
    """
    extract Dump_EM files from archive
    input:  start   --- start time in format of mm/dd/yy
            stop    --- stop time in format of mm/dd/yy
    output: *Dump_EM* data in ./EM_data directory
            data    --- return data lists
    """
#
#--- get data from archive
#
    run_arc4gl(start, stop)
#
#--- move the data to EM_Data directory and return the list of the data extracted
#
    try:
        cmd = 'mv *Dump_EM* ' + exc_dir + 'EM_Data/.'
        os.system(cmd)

        cmd = 'gzip -d ' + exc_dir + 'EM_Data/*gz'
        os.system(cmd)

        cmd = 'ls ' + exc_dir + 'EM_Data/*Dump_EM*sto > ' + zspace
        os.system(cmd)

        f    = open(zspace, 'r')
        data = [line.strip() for line in f.readlines()]
        f.close()
        mcf.rm_file(zspace)
    except:
        data = []

    return  data

def get_data(msid, start, stop):
    """
    extract data for the given data period
    input:  msid    --- msid
            start   --- starting time in seconds from 1998.1.1
            stop    --- stopping time in seconds from 1998.1.1
    output: data    --- a list of msid values
    """
#
#--- extract data with dataseeker
#
    try:
        ecf.data_seeker(start, stop, msid)
        [col, tbdata] = ecf.read_fits_file('temp_out.fits')
        mcf.rm_file('temp_out.fits')
    except:
        return []

    time = tbdata.field('time')
#
#--- if the dataseeker's data is not filled for the given data period
#--- stop any farther data proccess
#
    if time[-1] < 0.95 * stop:
        data = []
    else:
        try:
            name = msid + '_avg'
            data = tbdata.field(name)
        except:
            data = []

    return data

def run_focal_temp_data(start, stop):
    """
    run focal temp script and create a plot, read a table
    input:  start   --- start time in seconds from 1998.1.1
            stop    --- stop time in seconds from 1998.1.1
    output: fcnt    --- number of peaks observed
            fdata   --- table input
    """

    cmd1 = "/usr/bin/env PERL5LIB="
    cmd2 = ' /usr/local/bin/perl ' + wdir + 'get_ftemp_data.perl ' + str(start) + ' ' +  str(stop)
    cmd  = cmd1 + cmd2
#
#--- run the focal temp script to extract data
#
    bash(cmd,  env=ascdsenv)

    mcf.rm_file('./test')

    cmd1 = "/usr/bin/env PERL5LIB="
    cmd2 = ' idl ./run_temp > out'
    cmd  = cmd1 + cmd2
#
#--- run the focal temp script to create a plot
#
    bash(cmd,  env=ascdsenv2)

    cmd = 'rm -rf ./*fits '
    os.system(cmd)

def run_bad_pix_and_photon(outdir):
    """
    run bad pixel table script and photon table scrit
    input:  outdir  --- output directory name 
    output: files in outdir: bad_pix_list, photons
    """

    cmd1 = "/usr/bin/env PERL5LIB="
    cmd2 =  'perl /data/mta4/MTA/bin/weekly_obs2html.pl 8 photons'
    cmd  = cmd1 + cmd2

#
#--- run the phonton script
#
    bash(cmd,  env=ascdsenv)
    mcf.rm_file(zspace)


    cmd2 = 'perl ' + tdir + 'read_bad_pix_new.perl'
    cmd  = cmd1 + cmd2

#
#--- run the bad pixel script
#
    bash(cmd,  env=ascdsenv)
    mcf.rm_file(zspace)

    cmd = 'mv photons bad_pix_list ' + outdir
    os.system(cmd)

def find_last_entry(dir, tail):
    """
    for a given file name (with a full path), extract date information
    input:  dir     --- the name of directory where the data are kept
            tail    --- a suffix of the data file
    output: [year, mon, day]
    """
#
#--- find the last file created
#
    cmd = 'ls ' + dir + '/*' + tail + '| tail -1 > ' + zspace
    os.system(cmd)
    file = open(zspace, 'r').read()
    mcf.rm_file(zspace)
#
#--- extract time part. assume that the file end <tail> and the time part is prepended to it
#
    atemp = re.split('/', file)
    for ent in atemp:
        chk = re.search(tail, ent)
        if chk is not None:
            btemp = re.split('_', ent)
            ldate = btemp[0]
            break
#
#--- assume that the time is in the format of yyyymmdd, e.g. 20140515
#
    year = int(float(ldate[0:4]))
    mon  = int(float(ldate[4:6]))
    day  = int(float(ldate[6:8]))
    return [year, mon, day]

def run_ftp(tdir, dlist, ftp_address = ftp_site, outdir = './'):
    """
    retrieve files from ftp site
    input:  tdir        --- location of ftp file under "ftp_site"
            dlist       --- a list of file names you want to retrieve
            ftp_address --- ftp address, default: ftp_site (see the top of this script)
            outdir      --- a directory name where you want to deposit files. default: './'
    output: retrieved files in outdir
            count       --- the number of files retrieved
    """
#
#--- open ftp connection
#
    ftp = FTP(ftp_address)
    ftp.login('anonymous', '[email protected]')
    ftp.cwd(tdir)
#
#--- check though the data
#
    count = 0
    for file in dlist:
        local_file = os.path.join(outdir, file)
        try:
            ftp.retrbinary('RETR %s' %file, open(local_file, 'wb').write) 
            count += 1
        except:
            pass
#
#--- checking whether the retrieved file is empty, if so, just remove it
#
        if os.stat(local_file)[6] == 0:
            mcf.rm_file(local_file)
    ftp.quit()

    return count 

def extract_acis_evt1(start, stop):
    """
    extract acis evt1 files
    input:  start   --- start time in the format of mm/dd/yy (e.g. 05/01/15)
            stop    --- sop time in the format of mm/dd/yy
    output: acisf*evt1.fits.gz
    """
#
#--- write  required arc4gl command
#
    line = 'operation=retrieve\n'
    line = line + 'dataset=flight\n'
    line = line + 'detector=acis\n'
    line = line + 'level=1\n'
#    line = line + 'version=last\n'
    line = line + 'filetype=evt1\n'
    line = line + 'tstart=' + start + '\n'
    line = line + 'tstop='  + stop  + '\n'
    line = line + 'go\n'
    f    = open(zspace, 'w')
    f.write(line)
    f.close()

    cmd1 = "/usr/bin/env PERL5LIB="
    cmd2 =  ' echo ' +  hakama + ' |arc4gl -U' + dare + ' -Sarcocc -i' + zspace
    cmd  = cmd1 + cmd2

    bash(cmd,  env=ascdsenv)
    mcf.rm_file(zspace)

def get_sca_data():
#
# NOTE: sca00 is not updated anymore and discoutinued. 
#
    """
    extract ephsca.fits data file from dataseeker
    input:  none
    output: ephsca.fits
    """
#
#--- create an empty "test" file
#
    mcf.rm_file('./test')
    fo = open('./test', 'w')
    fo.close()
#
#--- and run dataseeker
#
    cmd1 = '/usr/bin/env PERL5LIB='
    cmd2 = ' dataseeker.pl infile=test outfile=ephsca.fits search_crit="columns=_sca00_avg" '
    cmd3 = 'clobber=yes loginFile=/home/mta/loginfile'
    cmd = cmd1 + cmd2 + cmd3
    bash(cmd, env=ascdsenv)

    mcf.rm_file('./test')

    cmd = 'mv -f ephsca.fits /data/mta4/www/DAILY/mta_rad/.'
    os.system(cmd)

def selectTableData(file, colname, interval, outname, extension = 1, clobber='no'):

    """
    select data for a given colname and the condition and create a new table fits file
    Input:      file     --- input table fits file
                colname  --- column name
                interval --- the data interval inthe forma of <start>:<stop>
                outname  --- output file name
                clobber  --- overwrite the file if exists. if not given, 'no'


    """

    m1 = re.search('y', clobber)
    m2 = re.search('Y', clobber)
    if (m1 is not None) or (m2 is not None):
        mcf.rm_file(outname)

    t     = fits.open(file)
    tdata = t[extension].data
    
    atemp = re.split(':', interval)
    start = float(atemp[0])
    stop  = float(atemp[1])

    mask  = tdata.field(colname) >= start
    modt  = tdata[mask]
    mask  = modt.field(colname)  <= stop
    modt2 = modt[mask]

    header  = fits.getheader(file)
    data    = fits.BinTableHDU(modt2, header)
    data.writeto(outname)

def find_two_sigma_value(fits):
#
#-- make histgram
#
    cmd1 = "/usr/bin/env PERL5LIB="
    cmd2 = ' dmimghist infile=' + fits + '  outfile=outfile.fits hist=1::1 strict=yes clobber=yes'
    cmd  = cmd1 + cmd2
    bash(cmd,  env=ascdsenv)

    cmd1 = "/usr/bin/env PERL5LIB="
    cmd2 = ' dmlist infile=outfile.fits outfile=' + zspace + ' opt=data'
    cmd  = cmd1 + cmd2
    bash(cmd,  env=ascdsenv)

    
    f= open(zspace, 'r')
    data = [line.strip() for line in f.readlines()]
    f.close()
    mcf.rm_file(zspace)
#
#--- read bin # and its count rate
#
    hbin = []
    hcnt = []
    vsum = 0

    for ent in data:
        atemp = re.split('\s+|\t+', ent)
        if mcf.chkNumeric(atemp[0]):
            hbin.append(float(atemp[1]))
            val = int(atemp[4])
            hcnt.append(val)
            vsum += val

#
#--- checking one sigma and two sigma counts
#

    if len(hbin) > 0:
        v68= int(0.68 * vsum)
        v95= int(0.95 * vsum)
        v99= int(0.997 * vsum)
        sigma1 = -999
        sigma2 = -999
        sigma3 = -999
        acc= 0
        for i in range(0, len(hbin)):
            acc += hcnt[i]
            if acc > v68 and sigma1 < 0:
                sigma1 = hbin[i]
            elif acc > v95 and sigma2 < 0:
                sigma2 = hbin[i]
            elif acc > v99 and sigma3 < 0:
                sigma3 = hbin[i]
                break
    
        return (sigma1, sigma2, sigma3)
    
    else:
        return(0, 0, 0)

def get_cti_data(test=''):
    """
    get cti data
    input:  none
    output: cti_data.txt
    """
#
#--- get all data
#
    cmd = 'cat /data/mta/Script/ACIS/CTI/Data/Det_Data_adjust/mn_ccd* > ' + zspace
    os.system(cmd)
#
#--- check the previous data file exists. if so, move to save.
#
    if os.path.isfile('./cti_data.txt') and test == '':
        cmd = 'mv -f  cti_data.txt cti_data.txt~'
        os.system(cmd)
#
#--- sort and select out data
#
    cmd = 'sort -u -k 1,1 ' + zspace + ' -o cti_data.txt'
    os.system(cmd)
    mcf.rm_file(zspace)
#
#--- change the permission
#
    cmd = 'chmod 755 cti_data.txt'
    os.system(cmd)
    cmd = 'mv -f ./cti_data.txt /data/mta4/www/DAILY/mta_rad/'
    os.system(cmd)

def find_observation(start, stop, lev):
    """
    find information about observations in the time period
    input:  start   --- starting time in the format of <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss>
            stop    --- stoping time in the format of <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss>
            lev     --- data level
    output: acis_obs    --- a list of the observation infromation 
    """
#
#--- run arc5gl to get observation list
#
    run_arc5gl_browse(start, stop,lev, zspace)
#
#--- create obsid list
#
    data = read_file(zspace, remove=1)
    obsid_list = []
    for ent in data:
        mc = re.search('acisf', ent)
        if mc is not None:
            atemp = re.split('acisf', ent)
            btemp = re.split('_', atemp[1])
            obsid = btemp[0]
            mc    = re.search('N', obsid)
            if mc is not None:
                ctemp = re.split('N', obsid)
                obsid = ctemp[0]
            obsid_list.append(obsid)
#
#--- remove duplicate
#
    o_set = set(obsid_list)
    obsid_list = list(o_set)
#
#--- open database and extract data for each obsid
#
    save  = {}
    tlist = []
    for obsid in obsid_list:
        out = get_data_from_db(obsid)
        if out != NULL:
            [tsec, line] = out
            tlist.append(tsec)
            save[tsec] = line

    tlist.sort()

    mcf.rm_file('./acis_obs')
    fo = open('./acis_obs', 'w')
    for ent in tlist:
        fo.write(save[ent])

    fo.close()

def read_data(infile, remove=0):

    try:
        f    = open(infile, 'r')
        data = [line.strip() for line in f.readlines()]
        f.close()
    except:
        data = []

    if remove == 1:
        mcf.rm_file(infile)

    return data