def findColumns(col_names,req_cols,opt_cols=[]):
    """ findColumns finds the indices for the column names in the list of required columns
    indices = findColumns(col_names, req_cols)
    
    indices - a list of indices, each index maps to where in col_names the column is found and in the order of req_cols
    col_names - list of column names to be searched
    req_cols - list of names that should be in the first list
    """
    idx = []
    didx = dict()

    for r in req_cols:
        if r in col_names:
            didx[r] = col_names.index(r)
        else:
            apflog("%s Not found in column names from google spreadsheet" % (r) , level="Alert",echo=True)

    for r in opt_cols:
        if r in col_names:
            didx[r] = col_names.index(r)
            
    # hack to handle an error
    if req_cols[0] == "Star Name" and req_cols[0] not in didx.keys():
        didx[req_cols[0]] = 0
        apflog("Pasting 'Star Name' into column 0 of google spreadsheet" , level="Error",echo=True)

    return didx

    def run_focustel(self):
        """Runs the telescope focus routine."""
        el = self.tel['EL'].read(binary=True)
        cfspos = self.fspos.read(binary=True)
        crspos = self.rspos.read(binary=True)

        if abs(el - cfspos) < 2.5 or abs(el - crspos) < 2.5:
            apflog("Cannot focus, telescope too close to shutter", level="warn", echo=True)
            return False
           
        if self.test: 
            APFTask.waitFor(self.task, True, timeout=10)
            apflog("Test Mode: Would be running focus_telescope.",echo=True)
            return True
        else:
            apflog("Running focus_telescope routine.",echo=True)
            cmdpath = '/usr/local/lick/bin/robot/'
            cmd = os.path.join(cmdpath,'focus_telescope')
            result, code = cmdexec(cmd,cwd=os.path.curdir)
            if not result:
                apflog("focustel failed with code %d" % code, echo=True)
                expression="($apftask.FOCUSINSTR_STATUS != 0) and ($apftask.FOCUSINSTR_STATUS != 1) "
                if not APFTask.waitFor(self.task,True,expression=expression,timeout=30):
                    apflog("focus_telescope failed to exit" ,echo=True)
                return result
            return True

def dmtimemon(dmtime):
    if dmtime['populated'] == False:
        return
    try:
        APF.dmtime = dmtime
    except Exception, e:
        apflog("Exception in dmtimemon: %s" % (e), level='error')

 def run_centerup(self):
     cmdpath = '/usr/local/lick/bin/robot/'
     cmd = os.path.join(cmdpath,'centerup')
     result, code = cmdexec(cmd,cwd=os.path.curdir)
     if not result:
         apflog("centerup failed with code %d" % code, echo=True)
     return result

 def evening_star(self):
     """Aim the APF at the desired target. This calls prep-obs, slewlock, and focus-telescope. A workaround to relying on scriptobs."""
     if self.isOpen()[0] == False:
         apflog("APF is not open. Can't target a star while closed.",level='error',echo=True)
         return
     self.DMReset()
     # Call prep-obs
     apflog("Calling prep-obs.",echo=True)
     result, ret_code = cmdexec('prep-obs')
     if result == False:
         apflog("Prep-obs returned error code %d. Targeting object has failed." % (ret_code),level='error',echo=True)
         return
     self.DMReset()
     apflog("Slewing to lower el",echo=True)
     result, ret_code = cmdexec('slew -e 75')
     if result == False:
         apflog("Slew returned error code %d. Targeting object has failed." % (ret_code),level='error',echo=True)
         return
     # Slew to the specified RA and DEC, set guide camera settings, and centerup( Slewlock )
     # Focus the telescope?
     self.DMReset()
     if self.focusTel():
         return True
     else:
         return False

 def DMZero(self):
     try:
         if self.checkapf['DMTIME'].read(binary=True) < 1:
             APFLib.write(self.checkapf['DMTIME'], -1,timeout=10)
     except Exception, e:
         ostr = "Error: cannot touch DM Timer: %s " %( e)
         apflog(ostr,level='error',echo=True)

def getObserved(filename):
    """ getObserved parses a file to find the object names and times
    names, times = getObserved(filename)
    names - list of names, must be first column of file called filename
    times - times either as a timestamp in second column or a (hour,minute) tuple from a scriptobs line

    """
    obs = []
    times = []
    nobs = dict()
    try:
        f = open(filename, 'r')
    except IOError:
        apflog( "Couldn't open %s" % filename,level="warn",echo=True)
        return obs, times
    else: 
        for line in f:
            line = line.strip()
            if len(line) > 0:
                if line[0] == '#' or line == "":
                    pass
                else:
                    ls = line.split()
                    obs.append(ls[0])
                    if len(ls) > 15:
                        times.append( (int(ls[14].split('=')[1]), int(ls[15].split('=')[1])) )
                    else:
                        times.append(float(ls[1]))
            
    obs.reverse()
    times.reverse()
    return obs, times

 def calibrate(self, script, time):
     s_calibrate = os.path.join(ScriptDir,"calibrate")
     if self.test: 
         print "Test Mode: calibrate %s %s." % (script, time)
         APFTask.waitFor(self.task, True, timeout=10)
         return True
     if time == 'pre' or 'post':
         try:
             APFLib.write("apfmot.DEWARFOCRAW",ktl.read("apftask","FOCUSINSTR_LASTFOCUS",binary=True))
         except:
             apflog("Cannot read the last best fitting focus value or write the dewar focus value", level='error')
         if self.dewarfoc > 8600 or self.dewarfoc < 8400:
             apflog("Warning: The dewar focus is currently %d. This is outside the typical range of acceptable values." % (self.dewarfoc), level = "error", echo=True)
             return False
         apflog("Running calibrate %s %s" % (script, time), level = 'info')
         owner = self.apfschedule('OWNRHINT').read()        
         self.apfschedule('OWNRHINT').write('public')        
         
         cmd = '%s %s %s' % (s_calibrate,script, time)
         result, code = cmdexec(cmd,debug=True,cwd=os.getcwd())
         if not result:
             apflog("%s %s failed with return code %d" % (s_calibrate, script, code),echo=True)
         expression="($apftask.CALIBRATE_STATUS != 0) and ($apftask.CALIBRATE_STATUS != 1) "
         if not APFTask.waitFor(self.task,True,expression=expression,timeout=30):
             apflog("%s %s failed to exit" % (s_calibrate,script),echo=True)
         self.apfschedule('OWNRHINT').write(owner)        
             
         return result
     else:
         print "Couldn't understand argument %s, nothing was done." % time

def parseStarlist(starlist):
    """ Parse a scriptobs-compatible starlist for the scheduler.

    names, star_table, lines, stars = parseStarlist(starlist)
    starlist - a filename

    names - a list of stars in the starlist
    star_table - a numpy array
    lines - a list of strings that can be used for scriptobs input
    stars - a list of pyEphem objects 
    """
    names = []
    lines = []
    stars = []
    star_table = []
    try:
        f = open(starlist,'r')
    except IOError:
        apflog("Warning: Could not open %s. No target can be selected." % starlist,echo=True)
        return None
    else:
        for line in f:
            if not re.search("\A\#",line):
                ls = line.split()
                names.append(ls[0])
                row = []
                # RA value in radians
                row.append(getRARad(ls[1], ls[2], ls[3]))
                # Dec value in radians
                row.append(getDECRad(ls[4], ls[5], ls[6]))
                # PM RA
                row.append(float(ls[8].split('=')[-1]))
                # PM Dec
                row.append(float(ls[9].split('=')[-1]))
                # V mag
                row.append(float(ls[10].split('=')[-1]))
                # Exposure time
                row.append(float(ls[11].split('=')[-1]))
                # Desired Counts
                row.append(float(ls[16].split('=')[-1]))
                # Filler not used here
                row.append(0.)
                row.append(0.)
                # Number of exposures
                row.append(int(ls[19].split('=')[-1]))

                star_table.append(row)

                # Save the scriptobs line for later
                lines.append(line)

                # Generate a pyEphem object for this target
                star = ephem.FixedBody()
                star.name = ls[0]
                star._ra = ephem.hours(":".join([ls[1], ls[2], ls[3]]))
                star._dec = ephem.degrees(":".join([ls[4], ls[5], ls[6]]))
                stars.append(star)
            
    return names, np.array(star_table), lines, stars

def altwindmon(wx):
    if wx['populated'] == False:
        return
    try:
        downval = APF.down.read(binary=True)
    except Exception, e:
        apflog("Exception in altwindmon: %s" % (e), level='error')
        return

def windmon(wx):
    if wx['populated'] == False:
        return
    try:
        wvel = float(wx)
    except Exception, e:
        apflog("Exception in windmon: %s" % (e), level='error')
        return

def okmon(ok2open):
    if ok2open['populated'] == False:
        return
    try:
        ok = ok2open # historical
    except Exception, e:
        apflog("Exception in okmon for checkapf.OPEN_OK: %s" % (e), level='error')
        return

def okmon(ok2open):
    ok = ok2open.read(binary=True)
    if not checkapf['MOVE_PERM'].read(binary=False):
        ok = False
    if APF.wvel > windlim:
        apflog("Too windy!")
        ok = False
    # Also need to check for cloud cover. This could require moving this call below the condition checking code.
    APF.openOK = ok

 def DMReset(self):
     try:
         APFLib.write(self.checkapf['ROBOSTATE'], "master operating",timeout=10)
     except Exception, e:
         try:
             ukind = self.checkapf['USERKIND'].read()
         except:
             ukind = "Unknown"
         ostr = "Error: Cannot write to ROBOSTATE, USERKIND = %s, reason: %s" % (ukind,e)
         apflog(ostr,level='error',echo=True)

 def setTeqMode(self, mode):
     apflog("Setting TEQMode to %s" % mode)
     if self.test: 
         print "Would be setting TEQMode to %s" % mode
         return
     self.teqmode.write(mode,wait=False)
     result = self.teqmode.waitfor('== %s' % mode, timeout=60)
     if not result:
         apflog("Error setting the TEQMODE.")
         raise RuntimeError, "Couldn't set TEQ mode"

def instr_permit():
    instr_perm = ktl.read("checkapf","INSTR_PERM",binary=True)
    userkind = ktl.read("checkapf","USERKIND",binary=True)
    while not instr_perm or userkind != 3:
        apflog("Waiting for instrument permission to be true and userkind to be robotic")
        APFTask.waitfor(parent,True,expression="$checkapf.INSTR_PERM = true",timeout=600)
        APFTask.waitfor(parent,True,expression="$checkapf.USERKIND = robotic",timeout=600)
        instr_perm = ktl.read("checkapf","INSTR_PERM",binary=True)
        userkind = ktl.read("checkapf","USERKIND",binary=True)

    return True

    def run_autoexposure(self,ind=5):
        cmdpath = '/usr/local/lick/bin/robot/'
        cmd = os.path.join(cmdpath,'autoexposure')
        istr = "%d" % (ind)
        cmdargs = cmd
#       cmdargs = cmd + " -i " + istr
        result, code = cmdexec(cmdargs,cwd=os.path.curdir)

#        result, code = cmdexec([cmd,istr],cwd=os.path.curdir)
        if not result:
            apflog("autoexposure failed with code %d" % code, echo=True)
        return result

 def focusTel(self):
     star = self.find_star()
     if not star:
         apflog("Cannot find star near current position!?",level='error',echo=True)
         return False
     apflog("Targeting telescope on %s" % star[0], echo=True)
     
     if self.slew(star):
         if self.run_autoexposure(ind=1):
             if self.run_centerup():
                 return self.run_focustel()
     return False

def update_googledex_lastobs(filename, sheetns=["2018B"],ctime=None,certificate='UCSC Dynamic Scheduler-5b98d1283a95.json'):
    """
        Update the online googledex lastobs column assuming things in filename have been observed.
        update_googledex_lastobs(filename, sheetn="The Googledex",time=None,certificate='UCSC Dynamic Scheduler-5b98d1283a95.json')

        filename - where the observations are logged
    """
    names, times = ObservedLog.getObserved(filename)
    if len(names) == 0:
        return
    if ctime is None:
        ctime = datetime.utcfromtimestamp(int(time.time()))
    

    for sheetn in sheetns:
        ws = get_spreadsheet(sheetn=sheetn,certificate=certificate)
        if ws:
            vals = ws.get_all_values()
        else:
            next
        col = vals[0].index("lastobs") 
        nobscol = vals[0].index("Nobs")
    
        for i, v in enumerate(vals):
            # Did we observe this target tonight?
            if v[0] in names:
                # We observed this target, so update the cell in the worksheet
                # update_cell(row, col, val) - col and row are 1 indexed
                otime = times[names.index(v[0])]
                if isinstance(otime,float):
                    t = datetime.utcfromtimestamp(otime)
                else:
                    hr, mn = otime
                    t = datetime(ctime.year, ctime.month, ctime.day, hr, mn)
                jd = float(ephem.julian_date(t))
                try:
                    pastdate = float(v[col])
                    try:
                        n = int(v[nobscol])
                    except:
                        n = 0
                    if jd > pastdate:
                        ws.update_cell(i+1, col+1, round(jd, 2) )
                        ws.update_cell(i+1, nobscol+1, n + 1 )

                except:
                    print (v[0], v[col])
                    ws.update_cell(i+1, col+1, round(jd,2) )
                
            apflog( "Updated %s" % (sheetn),echo=True)

    return

def countratemon(kcountrate):
     if kcountrate['populated'] == False:
         return

     try:
         ctr = float(kcountrate['binary'])
     except:
         apflog("Cannot read apfguide.countrate",level='warn',echo=True)
         return
     APF.countrate *=  (1.0*APF.ncountrate)/(APF.ncountrate+1)
     APF.countrate += ctr/(APF.ncountrate+1)
     APF.ncountrate += 1
     return