def process_task(task_processes_avail):
    for sem, min_priority, max_priority in task_processes_avail:
        log.debug('trying to acquire sem for min {} max {}'.format(min_priority, max_priority))
        if sem.acquire(block=False):
            log.debug('acquired semaphore')
            db_task = utils.mongo_retry(lambda: get_db().tasks.find_one({'status': 'queued', 'start_after': {'$lte': datetime.now()}, 'priority': {'$gte': min_priority, '$lte': max_priority}}, sort=[('priority', pymongo.ASCENDING), ('start_after', pymongo.ASCENDING)]))
            if db_task is None:
                log.debug('no task to process, releasing sem')
                sem.release()
                log.debug('sem released')
            else:
                log.debug('found task to process')
                break
        else:
            log.debug('did not acquire semaphore')
    else:
        return False

    id = db_task['_id']

    log.debug('trying to lock task')
    res = utils.mongo_retry(lambda: get_db().tasks.update({'_id': id, 'status': 'queued'}, {'$set': {'status': 'running', 'timeout_time': datetime.now() + timedelta(seconds=db_task['timeout'])}}))
    if res['n'] == 0:
        log.debug('another instance got the task first, releasing sem')
        # Another instance got to this task before us
        sem.release()
        log.debug('sem released')
        return True

    def process_db_task():
        task = Task.from_db(db_task)

        log.info('processing task: {}'.format(task))

        try:
            next_tasks = utils.ensure_list(utils.call_in_process(task.run, timeout=task.timeout.total_seconds()))
        except Exception:
            log.info('task {} failed: {}'.format(task, traceback.format_exc()))
            log.debug('running failure method')
            next_tasks = task_failed(task)
            log.debug('done failure method')
            status = 'failed'
        else:
            log.info('task succeeded: {}'.format(task))
            status = 'done'

        done_task(task, id, status, next_tasks)
        log.debug('releasing sem')
        sem.release()

    proc = multiprocessing.Process(target=process_db_task)
    proc.start()

    def join_process():
        proc.join()

    threading.Thread(target=join_process).start()

    return True

 def GET(self):
     web.header("Content-Type","text/html; charset=utf-8")
     node_id = web.input().get("node_id")
     nodes = get_db().query(models.RadNode).all()
     if node_id:
         groups = get_db().query(models.RadGroup)\
          .filter(models.RadGroup.node_id == node_id).all()
     else:
         groups = get_db().query(models.RadGroup).all()
     return render("group.html",nodes=nodes,node_id=node_id,groups=groups)   

 def GET(self):
     web.header("Content-Type","text/html; charset=utf-8")
     node_id = web.input().get("node_id")
     nodes = get_db().query(models.RadNode).all()
     if node_id:
         users = get_db().query(models.RadUser)\
          .filter(models.RadUser.node_id == node_id).all()
     else:
         users = get_db().query(models.RadUser).all()
     return render("user.html",nodes=nodes,node_id=node_id,users=users)   

def handle_pending_task():
    db_task = utils.mongo_retry(lambda: get_db().tasks.find_one({'status': 'next_tasks_pending'}))
    if db_task is None:
        return False

    log.debug('handling pending task: {}'.format(db_task['class']))
    insert_db_tasks(db_task['next_tasks'])
    utils.mongo_retry(lambda: get_db().tasks.update({'_id': db_task['_id']}, {'$set': {'status': db_task['next_status']}, '$unset': {'next_tasks': '', 'next_status': ''}}))
    log.debug('done handling pending task: {}'.format(db_task['class']))

    return True

def get_game_ids_db():
    """
    looks up game_ids from db
    """
    db = utils.get_db()
    game_docs = db.games.find({},{'espn_game_id':True})
    return [game['espn_game_id'] for game in game_docs]

def _insert_shots(game_id):
    db = utils.get_db()
    if db.shots.find({'espn_game_id':game_id}).count():
       return 
    shots = _get_shots(game_id)
    for shot in shots:
        db.shots.save(shot)

def _latest_game_day():
    db = utils.get_db()
    last_day = dateparser.parse('2012-10-30')
    games = db.games.find().sort('date',-1)
    if games.count() == 0:
        return None
    return str(games[0].get('date'))

    def do_put_blob(self, args):
        """put_blob [bucket] [size] [key]

        Create a blob with random content.

        size must be a number of bytes followed by an optional multiplier:

            K - 1024
            M - 1048576
            G - 1073741824

        key is optional
        """
        argv = args.split()
        if len(argv) == 2:
            bucket, size = argv
            key = None
        elif len(argv) == 3:
            bucket, size, key = argv
        else:
            self.do_help('put_blob')
            return

        if not re.match(r"\d+[KMG]?$", size):
            print "invalid size: {}".format(size)
            return
        if not size.endswith(("K", "M", "G")):
            num_bytes = size
        else:
            multiplier = {"K": 1024, "M": 1024 ** 2, "G": 1024 ** 3}[size[-1]]
            num_bytes = int(size[:-1]) * multiplier

        db = get_db(self.config)
        key = db.random_file(bucket, num_bytes, key)
        print "put {} ({} = {} bytes)".format(key, size, num_bytes)

def done_task(task, id, status, next_tasks):
    if len(next_tasks) == 0:
        update = {'$set': {'status': status}}
    else:
        next_tasks_for_db = [task.for_db() for task in next_tasks]
        update = {'$set': {'status': 'next_tasks_pending', 'next_status': status, 'next_tasks': next_tasks_for_db}}
    utils.mongo_retry(lambda: get_db().tasks.update({'_id': id, 'status': 'running'}, update))

def main():
    global CACHE
    LOGGER.info('Starting alarm-clock')

    while True:
        now = datetime.datetime.now()
        today = now.strftime('%a').lower()

        db = get_db()
        playlist = db['playlist']
        alarm_str = db['days'][today]
        
        if alarm_str is not None:
            hour, minute = alarm_str.split(':')

            alarm_hour = datetime.time(int(hour), int(minute))

            dt = datetime.datetime.combine(now, alarm_hour)

            if not CACHE.get(dt, False):
                if dt <= now:
                    res = play(playlist)
                    if res:
                        CACHE[dt] = True

        CACHE = trim_cache(CACHE)
        time.sleep(60)

 def POST(self):
     web.header("Content-Type","text/html; charset=utf-8")
     form = forms.nas_add_form()
     if not form.validates(): 
         return render("baseform.html",form=form,title="新增NAS设备",action="/nas/add")    
     else:
         db = get_db()
         if db.query(models.RadNas).filter(models.RadNas.ip_addr == form.d.ip_addr).count()>0:
             return errorpage("Nas %s 已经存在"%form.d.ip_addr)       
         try:
             nas = models.RadNas()
             nas.id = nextid()
             nas.ip_addr = form.d.ip_addr
             nas.name = form.d.name
             nas.auth_secret = form.d.auth_secret
             nas.acct_secret = form.d.acct_secret
             nas.vendor_id = form.d.vendor_id
             nas.time_type = form.d.time_type
             nas.status = form.d.status
             db.add(nas)
             db.commit()
             db.flush()
         except Exception,e:
             db.rollback()
             log.error("add nes error: %s"%str(e))
             return errorpage("新增Nas失败 %s"%str(e))
         raise web.seeother("/nas",absolute=True)

    def POST(self):
        web.header("Content-Type","text/html; charset=utf-8")
        form = forms.product_add_form()
        if not form.validates(): 
            return render("baseform.html",form=form,title="新增产品套餐",action="/product/add")    
        else:
            db = get_db()
            if db.query(models.RadProduct).filter(models.RadProduct.id == form.d.id).count()>0:
                return errorpage("产品编号重复")
            try:
                radproduct = models.RadProduct()
                radproduct.id = form.d.id
                radproduct.name = form.d.name
                radproduct.policy = form.d.policy
                radproduct.fee_num = form.d.fee_num
                radproduct.fee_price = int(Decimal(form.d.fee_price)*100)
                radproduct.concur_number = form.d.concur_number
                radproduct.bind_mac = form.d.bind_mac
                radproduct.bind_vlan = form.d.bind_vlan
                radproduct.bandwidth_code = form.d.bandwidth_code
                radproduct.input_max_limit = form.d.input_max_limit
                radproduct.output_max_limit = form.d.output_max_limit
                radproduct.input_rate_code = form.d.input_rate_code
                radproduct.output_rate_code = form.d.output_rate_code
                radproduct.domain_code = form.d.domain_code
                radproduct.status = 0
                db.add(radproduct)
                db.commit()
                db.flush()
            except Exception,e:
                db.rollback()
                log.error("add product error: %s"%str(e))
                return errorpage("产品新增失败 %s"%str(e))

            raise web.seeother("/product",absolute=True)             

def left_right():
    db = utils.get_db()

    left_f = shot_filters.merge_filters({'left':0})
    left_df = DataFrame(list(db.shots.find(left_f)))
    n_l = len(left_df)
    left_p = len(left_df[left_df['made']]) / float(n_l)
    left_ci = utils.ci(left_p, n_l)

    right_f = shot_filters.merge_filters({'right':0})
    right_df = DataFrame(list(db.shots.find(right_f)))
    n_r = len(right_df)
    right_p = len(right_df[right_df['made']]) / float(n_r)
    right_ci = utils.ci(right_p, n_r)

    # # hypothesis test: different sides of court
    # pooled_p = (left_p*n_l + right_p*n_r)/(n_r + n_l)
    # pooled_se = np.sqrt(pooled_p * (1-pooled_p) * ((1.0/n_l) + (1.0/n_r)) )
    # pooled_z_stat = (right_p - left_p) / pooled_se
    json.dump([{
            'name': 'Right',
            'value': right_p,
            'l': right_ci[0],
            'u': right_ci[1]
        },
        {
            'name': 'Left',
            'value': left_p,
            'l': left_ci[0],
            'u': left_ci[1]
        }], open('static/data/left-right.json','w'))

 def POST(self):
     web.header("Content-Type","text/html; charset=utf-8")
     form = forms.group_add_form()
     if not form.validates(): 
         return render("baseform.html",form=form,title="新增用户组",action="/group/add")    
     else:
         db = get_db()
         if db.query(models.RadGroup)\
              .filter(models.RadGroup.node_id == form.d.node_id)\
              .filter(models.RadGroup.group_id == form.d.group_id)\
              .count()>0:
             return errorpage("用户组编码重复")       
         try:
             group = models.RadGroup()
             group.node_id = form.d.node_id
             group.group_id = form.d.group_id
             group.group_name = form.d.group_name
             db.add(group)
             db.commit()
             db.flush()
         except Exception,e:
             db.rollback()
             log.error("add group error: %s"%str(e))
             return errorpage("新增用户组失败 %s"%str(e))
         raise web.seeother("/group",absolute=True)

def quarterly():
    db = utils.get_db()
    shots = list(db.shots.find())
    shots_df = DataFrame(shots)

    q1 = shots_df['qtr'] == '1'
    q2 = shots_df['qtr'] == '2'
    q3 = shots_df['qtr'] == '3'
    q4 = shots_df['qtr'] == '4'
    q1q2 = np.any([shots_df['qtr'] == '1', shots_df['qtr'] == '2'], 0)

    qtr_ps = {
            'Q1': [len(shots_df[np.all([q1, shots_df['made']], 0)]) / float(len(shots_df[q1])), len(shots_df[q1])],
            'Q2': [len(shots_df[np.all([q2, shots_df['made']], 0)]) / float(len(shots_df[q2])), len(shots_df[q2])],
            'Q3': [len(shots_df[np.all([q3, shots_df['made']], 0)]) / float(len(shots_df[q3])), len(shots_df[q3])],
            'Q4': [len(shots_df[np.all([q4, shots_df['made']], 0)]) / float(len(shots_df[q4])), len(shots_df[q4])],
            # 'q1q2': [len(shots_df[np.all([q1q2, shots_df['made']], 0)]) / float(len(shots_df[q1q2])), len(shots_df[q1q2])]
            }

    cis = {}
    for q in qtr_ps:
        cis[q] = utils.ci(qtr_ps[q][0], qtr_ps[q][1])

    json.dump([{
        'name': q,
        'value': qtr_ps[q][0],
        'l': cis[q][0],
        'u': cis[q][1]
        } for q in qtr_ps], open('static/data/quarterly.json','w'))

def insert_game_ids():
    """
    iterates through days and finds all espn game_ids from those days.
    inserts them into mongo db.
    """
    #TODO: pass in progress callback
    db = utils.get_db()
    
    url = "http://scores.espn.go.com/nba/scoreboard"
    game_ids = set()
    last = _latest_game_day()
    for dt in utils.day_gen(last):
        day_str = dt.strftime(DT_FORMAT)
        try:
            resp = requests.get(url,params={'date':day_str})
            cur_ids = re.findall('href="/nba/playbyplay\?gameId=(\d*)',resp.text)
            game_ids.update(cur_ids)

            for game_id in cur_ids:
                mongo_game_id = _gen_game_key(game_id)
                doc = {'date':dt,'espn_game_id':game_id,'_id':mongo_game_id}
                db.games.save(doc)

        except Exception as e:
            pdb.set_trace()
    
    return

def total_quarterly():
    db = utils.get_db()
    num_shots = []
    for i in range(1,5):
        params = {'quarter': '%i' % i}
        shots = db.shots.find(shot_filters.merge_filters(params))
        num_shots.append({'name': 'Q%i' % i, 'value': shots.count()})
    json.dump(num_shots, open('static/data/total_quarterly.json','w'))

def _select_espn_game_ids():
    """
    returns a set of all espn game ids already in the db
    """
    db = utils.get_db()
    docs = db.games.find({},{'espn_game_id':True})
    game_ids = set([doc['espn_game_id'] for doc in docs])
    return game_ids

 def GET(self,productid):
     web.header("Content-Type","text/html; charset=utf-8")
     form = forms.product_update_form()
     db = get_db()
     product = db.query(models.RadProduct).get(productid)
     form.fill(product)
     form.fee_price.set_value(product.fee_price/100.00)
     return render("baseform.html",form=form,title="修改产品套餐",action="/product/update/")   

 def do_list_buckets(self, args):
     db = get_db(self.config)
     try:
         buckets = db.get_buckets()
     except Exception as e:
         print e
     else:
         for bucket in buckets:
             print bucket