def dashboardAsync(self, app_id, exp_uid, args, ignore_result=False):
        """
        Run a task (task_name) on a set of args with a given app_id, and exp_uid.
        Waits for computation to finish and returns the answer unless ignore_result=True in which case its a non-blocking call.
        No guarantees about order of execution.

        Inputs: ::\n
            (string) app_id, (string) exp_id, (string) task_name, (json) args
        Outputs: ::\n
            task_name(app_id, exp_id, args)

        """
        submit_timestamp = utils.datetimeNow('string')
        domain = self.__get_domain_for_job(app_id+"_"+exp_uid)
        if next.constants.CELERY_ON:
            result = tasks.apply_dashboard.apply_async(args=[app_id,
                                                             exp_uid,
                                                             args,
                                                             submit_timestamp],
                                                       exchange='[email protected]'+domain,
                                                       routing_key='[email protected]'+domain)
            if ignore_result:
                return True
            else:
                return result.get(interval=0.001)
        else:
            result = tasks.apply_dashboard(app_id,exp_uid, args, submit_timestamp)
            if ignore_result:
                return True
            else:
                return result

    def getModel(self, exp_uid, args_json):
        try:
            args_dict = self.helper.convert_json(args_json)
            args_dict = verifier.verify(args_dict, self.reference_dict['getModel']['args'])
            alg_label = args_dict['args']['alg_label']
            args = self.butler.experiment.get(key='args')
            for algorithm in args['alg_list']:
                if alg_label == algorithm['alg_label']:
                    alg_id = algorithm['alg_id']

            myapp_response = self.call_app_fn(alg_label, alg_id, 'getModel', args_dict)

            myapp_response['exp_uid'] = exp_uid
            myapp_response['alg_label'] = alg_label
            # Log the response of the getModel in ALG-EVALUATION
            if args_dict['args']['logging']:
                alg_log_entry = {'exp_uid': exp_uid, 'alg_label':alg_label, 'task': 'getModel', 'timestamp': str(utils.datetimeNow())}
                alg_log_entry.update(myapp_response)
                self.butler.log('ALG-EVALUATION', alg_log_entry)
            return json.dumps({'args': myapp_response,
                               'meta': {'log_entry_durations':self.log_entry_durations,
                                        'timestamp': str(utils.datetimeNow())}}), True, ''
        except Exception, error:
            exc_type, exc_value, exc_traceback = sys.exc_info()
            full_error = str(traceback.format_exc())+'\n'+str(error)
            utils.debug_print("getModel Exception: " + full_error, color='red')
            log_entry = { 'exp_uid':exp_uid,'task':'getModel','error':full_error,'timestamp':utils.datetimeNow(),'args_json':args_json }
            self.butler.ell.log( self.app_id+':APP-EXCEPTION', log_entry  )
            traceback.print_tb(exc_traceback)
            return Exception(error)

    def applySyncByNamespace(self, app_id, exp_uid, task_name, args, namespace=None, ignore_result=False,time_limit=0):
        """
        Run a task (task_name) on a set of args with a given app_id, and exp_uid asynchronously. 
        Waits for computation to finish and returns the answer unless ignore_result=True in which case its a non-blocking call. 
        If this method is called a sequence of times with the same namespace (defaults to exp_uid if not set) it is guaranteed that they will execute in order, each job finishing before the next begins

        Inputs: ::\n
            (string) app_id, (string) exp_id, (string) task_name, (json) args

        """
        if namespace==None:
            namespace=exp_uid
        domain = self.__get_domain_for_job(app_id+"_"+exp_uid)
        job_uid = utils.getNewUID()
        submit_timestamp = utils.datetimeNow('string')
        if time_limit == 0:
            soft_time_limit = None
            hard_time_limit = None
        else:
            soft_time_limit = time_limit
            hard_time_limit = time_limit + .01
        result = tasks.apply_sync_by_namespace.apply_async(args=[app_id,exp_uid,task_name, args, namespace, job_uid, submit_timestamp, time_limit], exchange='[email protected]'+domain, routing_key='[email protected]'+domain,soft_time_limit=soft_time_limit,time_limit=hard_time_limit)
        if ignore_result:
            return True
        else:
            return result.get(interval=.001) 

def test_log(lapi):
    B = 'test_log'

    now = utils.datetimeNow()
    log_entry = {'a': 2, 'timestamp': now}

    lapi.log(B, log_entry)
    assert lapi._bucket(B).find_one({'timestamp': now}).get('a') == 2

    def applySyncByNamespace(self, app_id, exp_uid, task_name, args, namespace=None, ignore_result=False,time_limit=0):
        """
        Run a task (task_name) on a set of args with a given app_id, and exp_uid asynchronously. 
        Waits for computation to finish and returns the answer unless ignore_result=True in which case its a non-blocking call. 
        If this method is called a sequence of times with the same namespace (defaults to exp_uid if not set) it is guaranteed that they will execute in order, each job finishing before the next begins

        Inputs: ::\n
            (string) app_id, (string) exp_id, (string) task_name, (json) args

        """
        submit_timestamp = utils.datetimeNow('string')
        if namespace==None:
            namespace=exp_uid
        domain = self.__get_domain_for_job(app_id+"_"+exp_uid)
        num_queues = next.constants.CELERY_SYNC_WORKER_COUNT

        # assign namespaces to queues (with worker of concurrency 1) in round-robbin  
        try:
            namespace_cnt = int(self.r.get(namespace+"_cnt"))
        except:
            pipe = self.r.pipeline(True)
            while 1:
                try:
                    pipe.watch(namespace+"_cnt","namespace_counter")

                    if not pipe.exists(namespace+"_cnt"):
                        if not pipe.exists('namespace_counter'):
                            namespace_counter = 0
                        else:
                            namespace_counter = pipe.get('namespace_counter')
                        pipe.multi()
                        pipe.set(namespace+"_cnt",int(namespace_counter)+1)
                        pipe.set('namespace_counter',int(namespace_counter)+1)
                        pipe.execute()
                    else:
                        pipe.unwatch()
                    break
                except redis.exceptions.WatchError:
                    continue
                finally:
                    pipe.reset()
            namespace_cnt = int(self.r.get(namespace+"_cnt"))
        queue_number = (namespace_cnt % num_queues) + 1
        queue_name = 'sync_queue_'+str(queue_number)+'@'+domain

        job_uid = utils.getNewUID()
        if time_limit == 0:
            soft_time_limit = None
            hard_time_limit = None
        else:
            soft_time_limit = time_limit
            hard_time_limit = time_limit + .01
        result = tasks.apply_sync_by_namespace.apply_async(args=[app_id,exp_uid,task_name, args, namespace, job_uid, submit_timestamp, time_limit], queue=queue_name,soft_time_limit=soft_time_limit,time_limit=hard_time_limit)
        if ignore_result:
            return True
        else:
            return result.get(interval=.001) 

def test_get_logs_with_filter(lapi):
    B = 'test_get_logs_with_filter'

    now = utils.datetimeNow()
    log_entry = {'a': 2, 'timestamp': now}

    lapi.log(B, log_entry)
    retrieved_entry = lapi.get_logs_with_filter(B, {'timestamp': now})
    assert len(retrieved_entry) == 1
    assert retrieved_entry[0].get('a') == 2

    def most_current_embedding(self,app,butler,alg_label):
        """
        Description: Returns embedding in the form of a list of dictionaries, which is conveneint for downstream applications

        Expected input:
          (string) alg_label : must be a valid alg_label contained in alg_list list of dicts

        Expected output (in dict):
          plot_type : 'scatter2d_noaxis'
          (float) x_min : minimum x-value to display in viewing box
          (float) x_max : maximum x-value to display in viewing box
          (float) y_min : minimum y-value to display in viewing box
          (float) y_max : maximum y-value to display in viewing box
          (list of dicts with fields) data :
            (int) index : index of target
            (float) x : x-value of target
            (float) y : y-value of target
        """

        TargetManager = butler.targets
        item = app.getModel(json.dumps({'exp_uid':app.exp_uid, 'args':{'alg_label':alg_label}}))
        embedding = item['X']
        data = []
        x_min = numpy.float('inf')
        x_max = -numpy.float('inf')
        y_min = numpy.float('inf')
        y_max = -numpy.float('inf')
        for idx,target in enumerate(embedding):

            target_dict = {}
            target_dict['target'] = TargetManager.get_target_item(app.exp_uid, idx)
            target_dict['x'] = target[0] # this is what will actually be plotted,
            try:
                target_dict['y'] = target[1] # takes first two components, (could be replaced by PCA)
            except:
                target_dict['y'] = 0.
            target_dict['darray'] = target

            x_min = min(x_min,target[0])
            x_max = max(x_max,target[0])
            y_min = min(y_min,target[1])
            y_max = max(y_max,target[1])
            data.append(target_dict)

        return_dict = {'timestamp':str(utils.datetimeNow()),
                       'x_min':x_min, 'x_max':x_max, 'y_min':y_min, 'y_max':y_max, 'data':data,
                       'plot_type':'scatter2d_noaxis'}

        return return_dict

    def log(self,bucket_id,log_dict):
        """
        Saves log_dict to PermStore as an individual document for later recall.

        Inputs:
            (string) bucket_id, (dict with string values) log_dict

        Outputs:
            (bool) didSucceed, (string) message

        Usage: ::\n
            didSucceed,message = db.log(bucket_id,doc_uid)
        """
        timestamp = utils.datetimeNow()
        log_dict.update({ 'timestamp':timestamp })
        return self.permStore.setDoc(constants.logs_database_id,bucket_id,None,log_dict)

 def initExp(self, exp_uid, args_json):
     try:
         self.helper.ensure_indices(self.app_id,self.butler.db, self.butler.ell)
         args_dict = self.helper.convert_json(args_json)
         args_dict = verifier.verify(args_dict, self.reference_dict['initExp']['args'])
         args_dict['exp_uid'] = exp_uid # to get doc from db
         args_dict['start_date'] = utils.datetime2str(utils.datetimeNow())
         self.butler.admin.set(uid=exp_uid,value={'exp_uid': exp_uid, 'app_id':self.app_id, 'start_date':str(utils.datetimeNow())})
         self.butler.experiment.set(value={'exp_uid': exp_uid})
         args_dict['args'] = self.init_app(exp_uid, args_dict['args']['alg_list'], args_dict['args'])
         args_dict['git_hash'] = git_hash
         self.butler.experiment.set_many(key_value_dict=args_dict)
         return '{}', True, ''
     except Exception, error:
         exc_type, exc_value, exc_traceback = sys.exc_info()
         full_error = str(traceback.format_exc())+'\n'+str(error)
         utils.debug_print("initExp Exception: " + full_error, color='red')
         log_entry = { 'exp_uid':exp_uid,'task':'initExp','error':full_error,'timestamp':utils.datetimeNow(),'args_json':args_json }
         self.butler.ell.log( self.app_id+':APP-EXCEPTION', log_entry  )
         traceback.print_tb(exc_traceback)
         return '{}', False, str(error)

    def refresh_domain_hashes(self):

        # see what workers are out there
        worker_pings = None
        while worker_pings==None:
            try:
                # one could also use app.control.inspect().active_queues()
                worker_pings = app.control.inspect().ping()
            except:
                worker_pings = None
        worker_names = worker_pings.keys()
        domain_names_with_dups = [ item.split('@')[1] for item in worker_names]
        domain_names = list(set(domain_names_with_dups)) # remove duplicates!

        timestamp = utils.datetime2str(utils.datetimeNow())
        print "[ %s ] domains with active workers = %s" % (timestamp,str(domain_names))

        replicated_domains_hash = []
        for domain in domain_names:
            for i in range(self.num_replicas):
                replicated_domains_hash.append((domain, self.hash(domain+'_replica_'+str(i)), i))
        replicated_domains_hash = sorted( replicated_domains_hash, key = lambda x: x[1])  

        self.r.set('replicated_domains_hash',json.dumps(replicated_domains_hash))

  def initExp(self,exp_uid,args_json,db,ell):
    """
    initialize the project and necessary experiments 

    Expected input (in json structure with string keys):
      (int) n: number of arms
      (int) k: number of objects to display
      (float) failure_probability : confidence
      [optional] (list of dicts) alg_list : with fields (Defaults given by Info.get_app_default_alg_list)
            (string) alg_id : valid alg_id for this app_id
            (string) alg_label : unique identifier for algorithm (e.g. may have experiment with repeated alg_id's, but alg_labels must be unqiue, will also be used for plot legends
            [optional] (string) test_alg_label : must be one of the alg_label's in alg_list (Default is self)
      [optional] (dict) algorithm_management_settings : dictionary with fields (string) 'mode' and (dict) 'params'. mode in {'pure_exploration','explore_exploit','fixed_proportions'}. Default is 'fixed_proportions' and allocates uniform probability to each algorithm. If mode=fixed_proportions then params is a dictionary that contains the field 'proportions' which is a list of dictionaries with fields 'alg_label' and 'proportion' for all algorithms in alg_list. All proportions must be positive and sum to 1 over all algs in alg_list 
      [optional] (string) participant_to_algorithm_management : in {'one_to_one','one_to_many'}. Default is 'one_to_many'.
      [optional] (string) instructions
      [optional] (string) debrief
      
    Expected output:
      if error:
        return (JSON) '{}', (bool) False, (str) error_str
      else:
        return (JSON) '{}', (bool) True,''

    Usage:
      initExp_response_json,didSucceed,message = app.initExp(exp_uid,initExp_args_json)

    Example input:
      initExp_args_json = {"participant_to_algorithm_management": "one_to_many", "alg_list": [{"alg_label": "BR_LilUCB", "alg_id": "BR_LilUCB", "params": {}}], "algorithm_management_settings": {"params": {"proportions": [{"alg_label": "BR_LilUCB", "proportion": 1.0}]}, "mode": "fixed_proportions"}, "failure_probability": 0.01, "n": 10}

    Example output:
      initExp_response_json = {}
    """

    try:
      app_id = self.app_id

      # remove any reminants of an experiment if it exists
      didSucceed,message = db.delete_docs_with_filter('experiments_admin',{'exp_uid':exp_uid})
      didSucceed,message = db.delete_docs_with_filter(app_id+':experiments',{'exp_uid':exp_uid})
      didSucceed,message = db.delete_docs_with_filter(app_id+':queries',{'exp_uid':exp_uid})
      didSucceed,message = db.delete_docs_with_filter(app_id+':participants',{'exp_uid':exp_uid})
      didSucceed,message = db.delete_docs_with_filter(app_id+':algorithms',{'exp_uid':exp_uid})
      
      didSucceed,message = ell.delete_logs_with_filter(app_id+':APP-CALL',{'exp_uid':exp_uid})
      didSucceed,message = ell.delete_logs_with_filter(app_id+':APP-RESPONSE',{'exp_uid':exp_uid})
      didSucceed,message = ell.delete_logs_with_filter(app_id+':APP-EXCEPTION',{'exp_uid':exp_uid})
      didSucceed,message = ell.delete_logs_with_filter(app_id+':ALG-DURATION',{'exp_uid':exp_uid})
      didSucceed,message = ell.delete_logs_with_filter(app_id+':ALG-EVALUATION',{'exp_uid':exp_uid})

      # add indexes (only adds them if they do not already exist)
      didSucceed,message = db.ensure_index('experiments_admin',{'exp_uid':1})
      didSucceed,message = db.ensure_index(app_id+':experiments',{'exp_uid':1})
      didSucceed,message = db.ensure_index(app_id+':queries',{'query_uid':1})
      didSucceed,message = db.ensure_index(app_id+':queries',{'exp_uid':1})
      didSucceed,message = db.ensure_index(app_id+':queries',{'alg_uid':1})
      didSucceed,message = db.ensure_index(app_id+':queries',{'participant_uid':1})
      didSucceed,message = db.ensure_index(app_id+':participants',{'participant_uid':1})
      didSucceed,message = db.ensure_index(app_id+':participants',{'exp_uid':1})
      didSucceed,message = db.ensure_index(app_id+':algorithms',{'alg_uid':1})
      didSucceed,message = db.ensure_index(app_id+':algorithms',{'exp_uid':1})

      didSucceed,message = ell.ensure_index(app_id+':APP-CALL',{'exp_uid':1})
      didSucceed,message = ell.ensure_index(app_id+':APP-CALL',{'timestamp':1})
      didSucceed,message = ell.ensure_index(app_id+':APP-CALL',{'exp_uid':1,'timestamp':1})
      didSucceed,message = ell.ensure_index(app_id+':APP-CALL',{'exp_uid':1,'task':1})
      didSucceed,message = ell.ensure_index(app_id+':APP-RESPONSE',{'exp_uid':1})
      didSucceed,message = ell.ensure_index(app_id+':APP-RESPONSE',{'timestamp':1})
      didSucceed,message = ell.ensure_index(app_id+':APP-RESPONSE',{'exp_uid':1,'timestamp':1})
      didSucceed,message = ell.ensure_index(app_id+':APP-RESPONSE',{'exp_uid':1,'task':1})
      didSucceed,message = ell.ensure_index(app_id+':APP-EXCEPTION',{'exp_uid':1})
      didSucceed,message = ell.ensure_index(app_id+':APP-EXCEPTION',{'timestamp':1})
      didSucceed,message = ell.ensure_index(app_id+':APP-EXCEPTION',{'exp_uid':1,'timestamp':1})
      didSucceed,message = ell.ensure_index(app_id+':APP-EXCEPTION',{'exp_uid':1,'task':1})
      didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'exp_uid':1})
      didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'alg_uid':1})
      didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'timestamp':1})
      didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'exp_uid':1,'timestamp':1})
      didSucceed,message = ell.ensure_index(app_id+':ALG-DURATION',{'alg_uid':1,'task':1})
      didSucceed,message = ell.ensure_index(app_id+':ALG-EVALUATION',{'exp_uid':1})
      didSucceed,message = ell.ensure_index(app_id+':ALG-EVALUATION',{'alg_uid':1})
      didSucceed,message = ell.ensure_index(app_id+':ALG-EVALUATION',{'timestamp':1})
      didSucceed,message = ell.ensure_index(app_id+':ALG-EVALUATION',{'exp_uid':1,'timestamp':1})
      

      db.set('experiments_admin',exp_uid,'exp_uid',exp_uid)
      db.set('experiments_admin',exp_uid,'app_id',app_id)
      db.set('experiments_admin',exp_uid,'start_date',utils.datetime2str(utils.datetimeNow()))

      log_entry = { 'exp_uid':exp_uid,'task':'initExp','json':args_json,'timestamp':utils.datetimeNow() } 
      ell.log( app_id+':APP-CALL', log_entry  )

      # convert args_json to args_dict
      try:
        args_dict = json.loads(args_json)
      except:
        error = "%s.initExp input args_json is in improper format" % self.app_id
        return '{}',False,error

      # check for the fields that must be contained in args or error occurs
      necessary_fields = ['n','k','failure_probability']
#.........这里部分代码省略.........

  def getStats(self,exp_uid,args_json,db,ell):
    """
    Get statistics for the experiment and its algorithms

    Expected input (in json structure with string keys):
      (string) stat_id : identifier for the desired statistic
      (dict) params : dictionary of stat_id specific fields.

    See Next.utils.get_app_supported_stats(app_id) ResourceManager.get_app_supported_stats(app_id)
    for a description of each of the available stats and their inputs and outputs
    """

    try:
      app_id = self.app_id

      log_entry = { 'exp_uid':exp_uid,'task':'getStats','json':args_json,'timestamp':utils.datetimeNow() } 
      ell.log( app_id+':APP-CALL', log_entry  )

      # convert args_json to args_dict
      try:
        args_dict = json.loads(args_json)
      except:
        error = "%s.getStats input args_json is in improper format" % self.app_id
        return '{}',False,error

      # check for the fields that must be contained in args or error occurs
      necessary_fields = ['stat_id','params']
      for field in necessary_fields:
        try:
          args_dict[field]
        except KeyError:
          error = "%s.getStats input arguments missing field: %s" % (self.app_id,str(field)) 
          return '{}',False,error

      stat_id = args_dict['stat_id']
      params = args_dict['params']

      dashboard = TupleBanditsPureExplorationDashboard(db,ell)

      # input task
      if stat_id == "api_activity_histogram":
        task = params['task']
        activity_stats = dashboard.api_activity_histogram(self.app_id,exp_uid,task)
        stats = activity_stats

      # input Noneokay
      elif stat_id == "api_reportAnswer_activity_stacked_histogram":
        activity_stats = dashboard.api_reportAnswer_activity_stacked_histogram(self.app_id,exp_uid)
        stats = activity_stats

      # input task
      elif stat_id == "compute_duration_multiline_plot":
        task = params['task']
        compute_stats = dashboard.compute_duration_multiline_plot(self.app_id,exp_uid,task)
        stats = compute_stats

      # input task, alg_label
      elif stat_id == "compute_duration_detailed_stacked_area_plot":
        task = params['task']
        alg_label = params['alg_label']
        compute_detailed_stats = dashboard.compute_duration_detailed_stacked_area_plot(self.app_id,exp_uid,task,alg_label)
        stats = compute_detailed_stats

              # input alg_label
      elif stat_id == "response_time_histogram":
        alg_label = params['alg_label']
        response_time_stats = dashboard.response_time_histogram(self.app_id,exp_uid,alg_label)
        stats = response_time_stats
        
   # input alg_label
      elif stat_id == "network_delay_histogram":
        alg_label = params['alg_label']
        network_delay_stats = dashboard.network_delay_histogram(self.app_id,exp_uid,alg_label)
        stats = network_delay_stats


      elif stat_id == "most_current_ranking":
        alg_label = params['alg_label']
        stats = dashboard.most_current_ranking(self.app_id,exp_uid,alg_label)

      response_json = json.dumps(stats)

      log_entry = { 'exp_uid':exp_uid,'task':'getStats','json':response_json,'timestamp':utils.datetimeNow() } 
      ell.log( app_id+':APP-RESPONSE', log_entry  )

      return response_json,True,''
    except Exception, err:
      error = traceback.format_exc()
      log_entry = { 'exp_uid':exp_uid,'task':'getStats','error':error,'timestamp':utils.datetimeNow(),'args_json':args_json } 
      ell.log( app_id+':APP-EXCEPTION', log_entry  )
      return '{}',False,error

  def predict(self,exp_uid,args_json,db,ell):
    """
    Description: uses current model empirical estimates to forecast the ranking of the arms in order of likelhood of being the best from most to least likely

    Expected input:
      (string) predict_id : 'arm_ranking'
      (dict) params : dictionary with fields
          (string) alg_label : describes target algorithm to use

    Expected output (in json structure):
      (list of dicts with fields):
        (int) index : index of target
        (int) rank : rank that the algorithm assigns to index (rank 0 is most likely the best arm)
    """

    try:
      app_id = self.app_id

      log_entry = { 'exp_uid':exp_uid,'task':'predict','json':args_json,'timestamp':utils.datetimeNow() } 
      ell.log( app_id+':APP-CALL', log_entry  )

      # convert args_json to args_dict
      try:
        args_dict = json.loads(args_json)
      except:
        error = "%s.predict failed to convert input args_json due to improper format" %(self.app_id) 
        return '{}',False,error

      # check for the fields that must be contained in args or error occurs
      necessary_fields = ['predict_id','params']
      for field in necessary_fields:
        try:
          args_dict[field]
        except KeyError:
          error = "%s.predict input arguments missing field: %s" % (self.app_id,str(field)) 
          return '{}',False,error

      predict_id = args_dict['predict_id']
      params = args_dict['params']

      if predict_id == "arm_ranking":

        alg_label = params['alg_label']

        # get list of algorithms associated with project
        alg_list,didSucceed,message = db.get(app_id+':experiments',exp_uid,'alg_list')

        # get alg_id
        for algorithm in alg_list:
          if alg_label == algorithm['alg_label']:
            alg_id = algorithm['alg_id']
            alg_uid = algorithm['alg_uid']
            num_reported_answers,didSucceed,message = db.get(app_id+':experiments',exp_uid,'num_reported_answers_for_'+alg_uid)
            if type(num_reported_answers)!=int:
              num_reported_answers=0


        # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
        rc = ResourceClient(app_id,exp_uid,alg_uid,db)

        # get specific algorithm to make calls to 
        alg = utils.get_app_alg(self.app_id,alg_id)

        # call getQuery
        scores,precisions,dt = utils.timeit(alg.predict)(resource=rc)

        log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'predict','duration':dt } 
        log_entry_durations.update( rc.getDurations() )
        meta = {'log_entry_durations':log_entry_durations}

        import numpy
        ranks = (-numpy.array(scores)).argsort().tolist()

        n = len(scores)
        indexes = numpy.array(range(n))[ranks]
        scores = numpy.array(scores)[ranks]
        precisions = numpy.array(precisions)[ranks]
        ranks = range(n)

        targets = []
        for index in range(n):
          targets.append( {'index':indexes[index],'rank':ranks[index],'score':scores[index],'precision':precisions[index]} )

        log_entry = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'timestamp':utils.datetimeNow() } 
        log_entry.update( {'targets':targets,'num_reported_answers':num_reported_answers} )

        ell.log( app_id+':ALG-EVALUATION', log_entry  )

      response_args_dict = {}
      args_out = {'args':response_args_dict,'meta':meta}
      predict_json = json.dumps(args_out)

      log_entry = { 'exp_uid':exp_uid,'task':'predict','json':predict_json,'timestamp':utils.datetimeNow() } 
      ell.log( app_id+':APP-RESPONSE', log_entry  )

      return predict_json,True,''
    except Exception, err:
      error = traceback.format_exc()
      log_entry = { 'exp_uid':exp_uid,'task':'predict','error':str(error),'timestamp':utils.datetimeNow(),'args_json':args_json }  
      didSucceed,message = ell.log( app_id+':APP-EXCEPTION', log_entry  )
#.........这里部分代码省略.........

  def reportAnswer(self,exp_uid,args_json,db,ell):
    """
    reporting back the reward of pulling the arm suggested by getQuery

    Expected input:
      (str) query_uid : unique identifier of query
      (int) index_winner : index of arm must be one of the indices given by getQuery

    Expected output (comma separated): 
      if error:
        return (JSON) '{}', (bool) False, (str) error
      else:
        return (JSON) '{}', (bool) True,''

    Usage:
      reportAnswer_args_json,didSucceed,message = app.reportAnswer(exp_uid,reportAnswer_args_json)

    Example input:
      reportAnswer_args_json = {"query_uid": "4d02a9924f92138287edd17ca5feb6e1", "index_winner": 8}

    Example output:
      reportAnswer_response_json = {}
    """

    try:
      app_id = self.app_id

      log_entry = { 'exp_uid':exp_uid,'task':'reportAnswer','json':args_json,'timestamp':utils.datetimeNow() } 
      ell.log( app_id+':APP-CALL', log_entry  )

      # convert args_json to args_dict
      try:
        args_dict = json.loads(args_json)
      except:
        error = "%s.reportAnswer input args_json is in improper format" % self.app_id
        return '{}',False,error

      # check for the fields that must be contained in args or error occurs
      necessary_fields = ['index_winner','query_uid']
      for field in necessary_fields:
        try:
          args_dict[field]
        except KeyError:
          error = "%s.reportAnswer input arguments missing field: %s" % (self.app_id,str(field)) 
          return '{}',False,error

      # get list of algorithms associated with project
      alg_list,didSucceed,message = db.get(app_id+':experiments',exp_uid,'alg_list')

      # get alg_id
      query_uid = args_dict['query_uid']
      alg_uid,didSucceed,message = db.get(app_id+':queries',query_uid,'alg_uid')
      for algorithm in alg_list:
        if alg_uid == algorithm['alg_uid']:
          alg_id = algorithm['alg_id']
          alg_label = algorithm['alg_label']
          num_reported_answers,didSucceed,message = db.increment(app_id+':experiments',exp_uid,'num_reported_answers_for_'+alg_uid)

      # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
      rc = ResourceClient(app_id,exp_uid,alg_uid,db)

      # get specific algorithm to make calls to 
      alg = utils.get_app_alg(self.app_id,alg_id)

      # get targets associated with the specific query
      targets,didSucceed,message = db.get(app_id+':queries',query_uid,'target_indices')

      target_indices = []
      for target in targets:
        target_indices.append(target['index'])

      # get the index winner
      index_winner = args_dict['index_winner']

      # update query doc
      timestamp_query_generated,didSucceed,message = db.get(app_id+':queries',query_uid,'timestamp_query_generated')
      datetime_query_generated = utils.str2datetime(timestamp_query_generated)
      timestamp_answer_received = args_dict.get('meta',{}).get('timestamp_answer_received',None)
      if timestamp_answer_received == None:
        datetime_answer_received = datetime_query_generated
      else:
        datetime_answer_received = utils.str2datetime(timestamp_answer_received)
      delta_datetime = datetime_answer_received - datetime_query_generated
      round_trip_time = delta_datetime.seconds + delta_datetime.microseconds/1000000.
      response_time = float(args_dict.get('response_time',0.))
      db.set(app_id+':queries',query_uid,'response_time',response_time)
      db.set(app_id+':queries',query_uid,'network_delay',round_trip_time-response_time)
      db.set(app_id+':queries',query_uid,'index_winner',index_winner)

      # call reportAnswer
      didSucceed,dt = utils.timeit(alg.reportAnswer)(resource=rc,targets=target_indices,index_winner=index_winner)

      log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'reportAnswer','duration':dt } 
      log_entry_durations.update( rc.getDurations() )
      meta = {'log_entry_durations':log_entry_durations}

      # calling predict 
      ###############
      predict_id = 'arm_ranking'
      params = {'alg_label':alg_label}
#.........这里部分代码省略.........

  def getQuery(self,exp_uid,args_json,db,ell):
    """
    A request to ask which k arms to duel next

    Expected input (in jsonstructure with string keys):
      (int) k : number of objects to display
      [optional] (string) participant_uid :  unique identifier of session for a participant answering questions (that is, an email address is not good enough as the participant could participate in multiple exp_uids so it would not be unique against all experiments), if key non-existant particpant_uid is assigned as exp_uid. 

    Expected output (in json structure with string keys):
      (list) target_indices : list of k target indexes e.g. [ (int) target_index_1, ... , (int) target_index_k ]
      (str) query_uid : unique identifier of query (used to look up for reportAnswer)

    Usage: 
      getQuery_response_json,didSucceed,message = app.getQuery(exp_uid,getQuery_args_json)

    Example input:
      getQuery_args_json = {"k": 3, "participant_uid": "0077110d03cf06b8f77d11acc399e8a7"}

    Example output:
      getQuery_response_json = {"query_uid": "4d02a9924f92138287edd17ca5feb6e1", "target_indices": [ 3, 6, 9 ]

    """
    try: 
      app_id = self.app_id

      log_entry = { 'exp_uid':exp_uid,'task':'getQuery','json':args_json,'timestamp':utils.datetimeNow() } 
      ell.log( app_id+':APP-CALL', log_entry  )

      # convert args_json to args_dict
      try:
        args_dict = json.loads(args_json)
      except:
        error = "%s.initExp input args_json is in improper format" % self.app_id
        return '{}',False,error

      # get list of algorithms associated with project
      alg_list,didSucceed,message = db.get(app_id+':experiments',exp_uid,'alg_list')
      alg_label_to_alg_id = {}
      alg_label_to_alg_uid = {}
      for algorithm in alg_list:
        alg_label_to_alg_id[ algorithm['alg_label'] ] = algorithm['alg_id']
        alg_label_to_alg_uid[ algorithm['alg_label'] ] = algorithm['alg_uid']

      algorithm_management_settings,didSucceed,message = db.get(app_id+':experiments',exp_uid,'algorithm_management_settings')

      # ASSIGN ALGORITHM TO PARTICIPANT
      if 'participant_uid' in args_dict:
        participant_uid = args_dict['participant_uid']
      else:
        participant_uid = exp_uid

      participant_doc_exists,didSucceed,message = db.exists(app_id+':participants',participant_uid,'participant_uid')
      first_participant_query = not participant_doc_exists
      if first_participant_query:
        db.set(app_id+':participants',participant_uid,'participant_uid',participant_uid)
        db.set(app_id+':participants',participant_uid,'exp_uid',exp_uid)

      participant_to_algorithm_management,didSucceed,message = db.get(app_id+':experiments',exp_uid,'participant_to_algorithm_management')
      if (participant_uid==exp_uid) or (participant_to_algorithm_management=='one_to_many') or (first_participant_query):

        if algorithm_management_settings['mode']=='fixed_proportions':
          proportions_list = algorithm_management_settings['params']['proportions']
          prop = [ prop_item['proportion'] for prop_item in proportions_list ]
          prop_item = numpy.random.choice(alg_list,p=prop)
        else:
          raise Exception('algorithm_management_mode : '+algorithm_management_settings['mode']+' not implemented')
        alg_id = alg_label_to_alg_id[ prop_item['alg_label'] ] 
        alg_uid = alg_label_to_alg_uid[ prop_item['alg_label'] ]
        alg_label = prop_item['alg_label']
        
        if (first_participant_query) and (participant_to_algorithm_management=='one_to_one'):
          db.set(app_id+':participants',participant_uid,'alg_id',alg_id)
          db.set(app_id+':participants',participant_uid,'alg_uid',alg_uid)

      elif (participant_to_algorithm_management=='one_to_one'):
        # If here, then alg_uid should already be assigned in participant doc
        alg_id,didSucceed,message = db.get(app_id+':participants',participant_uid,'alg_id')
        alg_uid,didSucceed,message = db.get(app_id+':participants',participant_uid,'alg_uid')
      else:
        raise Exception('participant_to_algorithm_management : '+participant_to_algorithm_management+' not implemented')

      # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
      rc = ResourceClient(app_id,exp_uid,alg_uid,db)

      # get specific algorithm to make calls to 
      alg = utils.get_app_alg(self.app_id,alg_id)

      # call getQuery
      targets,dt = utils.timeit(alg.getQuery)(resource=rc)

      # update targets
      target_list = []
      for target in targets:
        target_list.append({'index':target})

      # check for context
      context_type,didSucceed,message = db.get(app_id+':experiments',exp_uid,'context_type')
      context,didSucceed,message = db.get(app_id+':experiments',exp_uid,'context')

      # log
#.........这里部分代码省略.........

  def daemonProcess(self,exp_uid,args_json,db,ell):
    try:

      app_id = self.app_id

      log_entry = { 'exp_uid':exp_uid,'task':'daemonProcess','json':args_json,'timestamp':utils.datetimeNow() } 
      ell.log( app_id+':APP-CALL', log_entry  )

      # convert args_json to args_dict
      try:
        args_dict = json.loads(args_json)
      except:
        error = "%s.daemonProcess input args_json is in improper format" % self.app_id
        return '{}',False,error

      # check for the fields that must be contained in args or error occurs
      necessary_fields = ['alg_uid','daemon_args']
      for field in necessary_fields:
        try:
          args_dict[field]
        except KeyError:
          error = "%s.daemonProcess input arguments missing field: %s" % (self.app_id,str(field)) 
          return '{}',False,error


      alg_daemon_args = args_dict['daemon_args']
      alg_uid = args_dict['alg_uid']
      alg_id,didSucceed,message = db.get(app_id+':algorithms',alg_uid,'alg_id')

      # get sandboxed database for the specific app_id,alg_id,exp_uid - closing off the rest of the database to the algorithm
      rc = ResourceClient(app_id,exp_uid,alg_uid,db)

      # get specific algorithm to make calls to 
      alg = utils.get_app_alg(self.app_id,alg_id)

      didSucceed,dt = utils.timeit(alg.daemonProcess)(resource=rc,daemon_args_dict=alg_daemon_args)
      
      log_entry_durations = { 'exp_uid':exp_uid,'alg_uid':alg_uid,'task':'daemonProcess','duration':dt } 
      log_entry_durations.update( rc.getDurations() )
      meta = {'log_entry_durations':log_entry_durations}

      daemon_message = {}
      args_out = {'args':daemon_message,'meta':meta}
      response_json = json.dumps(args_out)

      log_entry = { 'exp_uid':exp_uid,'task':'daemonProcess','json':response_json,'timestamp':utils.datetimeNow() } 
      ell.log( app_id+':APP-RESPONSE', log_entry  )

      return response_json,True,''

    except Exception, err:
      error = traceback.format_exc()
      log_entry = { 'exp_uid':exp_uid,'task':'daemonProcess','error':error,'timestamp':utils.datetimeNow(),'args_json':args_json } 
      ell.log( app_id+':APP-EXCEPTION', log_entry  )
      return '{}',False,error

  def getQuery(self,exp_uid,args_json,db,ell):
    """
    A request to ask which two arms to duel next

    Expected input (in jsonstructure with string keys):
      [optional] (string) participant_uid :  unique identifier of session for a participant answering questions (that is, an email address is not good enough as the participant could participate in multiple exp_uids so it would not be unique against all experiments), if key non-existant particpant_uid is assigned as exp_uid. 

    Expected output (in json structure with string keys):
      (list) target_indices : list that stores dictionary of targets with fields:
            { 
              (int) index : the index of the target of relevance
              (str) label : in {'left','right'} for display
              (int) flag : integer for algorithm's use
            }
      (str) query_uid : unique identifier of query (used to look up for processAnswer)
    """
    try: 
      app_id = self.app_id

      log_entry = { 'exp_uid':exp_uid,'task':'getQuery','json':args