def mutate(crossed,prob,maxLength,threshold = 1e6):


	for k,history in enumerate(crossed):
		
		for i,ele in enumerate(history):
			
			if util.flipCoin(prob):
				#if util.flipCoin(threshold * 1./fitness(ele,s_belief,t_belief,source_M, M_proj))
				if util.flipCoin(0.8):
					if util.flipCoin(0.5):
						mutated = list(ele)
						mutated[0] = random.choice(Actions)
						crossed[k][i] = tuple(mutated)
					else:
						mutated = list(ele)
						mutated[1] = Obs[np.random.choice(range(len(Obs)),1,p=Obs_p)[0]]
						crossed[k][i] = tuple(mutated)
				else:
					mutated = list(ele)
					mutated[0] = random.choice(Actions)
					mutated[1] = Obs[np.random.choice(range(len(Obs)),1,p=Obs_p)[0]]
					crossed[k][i] = tuple(mutated)
		if util.flipCoin(prob):
			if util.flipCoin(0.5) and len(history) < maxLength:
				mutated = [0,0]
				#mutated[0] = random.choice(Actions)
				#mutated[1] = Obs[np.random.choice(range(len(Obs)),1,p=Obs_p)[0]]
				mutated =  history[-1]
				crossed[k] = history + [tuple(mutated)] 

			elif len(history)>=maxLength-1:
				crossed[k].pop()
	return crossed

def generateVPIHuntersBoard(seed=None):
    width = 11
    height = 11
    foodHouseLeft = util.flipCoin(PROB_FOOD_LEFT)

    layoutTextGrid = [[' ' for _ in xrange(width)] for _ in xrange(height)]
    layoutTextGrid[0] = ['%' for _ in xrange(width)]
    layoutTextGrid[-1] = layoutTextGrid[0][:]
    for i in xrange(height):
        layoutTextGrid[i][0] = layoutTextGrid[i][-1] = '%'
    possibleLocations = pickPossibleLocations(width, height)
    # (foodX, foodY), (ghostX, ghostY) = tuple(random.sample(possibleLocations, 2))

    bottomLeft, topLeft, bottomRight, topRight = tuple(possibleLocations)

    foodX, foodY = topLeft
    ghostX, ghostY = topRight
    if not util.flipCoin(PROB_FOOD_LEFT):
        (foodX, foodY), (ghostX, ghostY) = (ghostX, ghostY), (foodX, foodY)

    layoutTextGrid[-foodY-1][foodX] = '.'
    layoutTextGrid[-ghostY-1][ghostX] = 'G'
    for foodWallX, foodWallY in buildHouseAroundCenter(foodX, foodY):
        if util.flipCoin(PROB_FOOD_RED):
            layoutTextGrid[-foodWallY-1][foodWallX] = 'R'
        else:
            layoutTextGrid[-foodWallY-1][foodWallX] = 'B'
    for ghostWallX, ghostWallY in buildHouseAroundCenter(ghostX, ghostY):
        if util.flipCoin(PROB_GHOST_RED):
            layoutTextGrid[-ghostWallY-1][ghostWallX] = 'R'
        else:
            layoutTextGrid[-ghostWallY-1][ghostWallX] = 'B'
    layoutTextGrid[5][5] = 'P'
    layoutTextRowList = [''.join(row) for row in layoutTextGrid]
    return layoutTextRowList

  def getAction(self, state):
    """
      Compute the action to take in the current state.  With
      probability self.epsilon, we should take a random action and
      take the best policy action otherwise.  Note that if there are
      no legal actions, which is the case at the terminal state, you
      should choose None as the action.

      HINT: You might want to use util.flipCoin(prob)
      HINT: To pick randomly from a list, use random.choice(list)
    """
    # Pick Action
    legalActions = self.getLegalActions(state)
    action = None
    "*** YOUR CODE HERE ***"
    if len(legalActions) == 0:
      return None

    # pick random one of the legal actions if
    # flipCoin returns true (exploration)
    if util.flipCoin(self.epsilon):
      return random.choice(legalActions)

    # otherwise, pick the best move determined by the policy (exploitation)
    return self.getPolicy(state)

    def getAction(self, state):
        """
          Compute the action to take in the current state.  With
          probability self.epsilon, we should take a random action and
          take the best policy action otherwise.  Note that if there are
          no legal actions, which is the case at the terminal state, you
          should choose None as the action.

          HINT: You might want to use util.flipCoin(prob)
          HINT: To pick randomly from a list, use random.choice(list)
        """
        # Pick Action
        legal_actions = self.getLegalActions(state)
        action = None
        "*** YOUR CODE HERE ***"
        if len(legal_actions) > 0:
            if util.flipCoin(self.epsilon):
                action = min([(self.state_uses[(state, act)], act) for act in legal_actions])[1]
                print action
                if not self.state_uses[(state, action)]:
                    self.state_uses[(state, action)] = 0
                else:
                    self.state_uses[(state, action)] += 1
            else:
                action = self.getPolicy(state)
                if not self.state_uses[(state, action)]:
                    self.state_uses[(state, action)] = 0
                else:
                    self.state_uses[(state, action)] += 1

        return action

    def getAction(self, state):
        """
          Compute the action to take in the current state.  With
          probability self.epsilon, we should take a random action and
          take the best policy action otherwise.  Note that if there are
          no legal actions, which is the case at the terminal state, you
          should choose None as the action.

          HINT: You might want to use util.flipCoin(prob)
          HINT: To pick randomly from a list, use random.choice(list)
        """
        # Pick Action
        legalActions = self.getLegalActions(state)
        action = None
        "*** YOUR CODE HERE ***"

        # comprobamos si hay legal actions, sino, retornamos None
        if not legalActions:
            return action

        # lanzamos la moneda con epsilon para decidir que accion retornamos, al azar o best policy
        if util.flipCoin(self.epsilon):
            # retornamos una accion al azar, si no devuelve nada retornamos None
            return random.choice(legalActions) or None

        # retornamos bestPolicy
        return self.getPolicy(state)

    def getAction(self, state):
        """
          Compute the action to take in the current state.  With
          probability self.epsilon, we should take a random action and
          take the best policy action otherwise.  Note that if there are
          no legal actions, which is the case at the terminal state, you
          should choose None as the action.

          HINT: You might want to use util.flipCoin(prob)
          HINT: To pick randomly from a list, use random.choice(list)
        """
        # Pick Action
        legalActions = self.getLegalActions(state)
        action = None
        "*** YOUR CODE HERE ***"

        #if terminal state return None
        if len(legalActions)==0:
            return None
        #check random true or false
        
        randomOrNot= util.flipCoin(self.epsilon)
        if  randomOrNot: 
            #Chose east, west, north, south? how do I get the list? 
            return   random.choice(legalActions)
          
        else: 
            #best policy action get policy or compute action from q values? 
            return self.computeActionFromQValues(state)
        
        util.raiseNotDefined()

    def getAction(self, state):
        """
          Compute the action to take in the current state.  With
          probability self.epsilon, we should take a random action and
          take the best policy action otherwise.  Note that if there are
          no legal actions, which is the case at the terminal state, you
          should choose None as the action.

          HINT: You might want to use util.flipCoin(prob)
          HINT: To pick randomly from a list, use random.choice(list)
        """
        # Pick Action
        legalActions = self.getLegalActions(state)
        action = None
        "*** YOUR CODE HERE ***"
        action = self.computeActionFromQValues(state)
        legal_actions=self.getLegalActions(state)
        if len(legal_actions)<=1:
            return action
        #suboptimal_actions.remove(action)


        # if state not in self.visit:
        #     self.visit[state]=0
        # self.visit[state]+=1

        if util.flipCoin(self.epsilon):#/self.visit[state]
            return random.choice(legal_actions)
        return action

  def getAction(self, state):
    """
      Compute the action to take in the current state.  With
      probability self.epsilon, we should take a random action and
      take the best policy action otherwise.  Note that if there are
      no legal actions, which is the case at the terminal state, you
      should choose None as the action.

      HINT: You might want to use util.flipCoin(prob)
      HINT: To pick randomly from a list, use random.choice(list)
    """
    # Pick Action
    legalActions = self.getLegalActions(state)
    action = None
    "*** YOUR CODE HERE ***"
    if len(legalActions) == 0:
        #print "No legal actions"
        action = None
    elif util.flipCoin(self.epsilon):
        #print "Random Choice of Action"
        action = random.choice(legalActions)
    else:
        #print "Choice of action based on Policy"
        action = self.getPolicy(state)
    #print "Action:", action
    return action

    def getAction(self, state):
        """
          Compute the action to take in the current state.  With
          probability self.epsilon, we should take a random action and
          take the best policy action otherwise.  Note that if there are
          no legal actions, which is the case at the terminal state, you
          should choose None as the action.

          HINT: You might want to use util.flipCoin(prob)
          HINT: To pick randomly from a list, use random.choice(list)
        """
        # Pick Action
        legal_actions = self.getLegalActions(state)
        action = None
        "*** YOUR CODE HERE ***"
        # util.raiseNotDefined()

        # return action
        if not legal_actions:
            return None

        # using flip coin fuction for adding randomness when
        # choosing action
        if util.flipCoin(self.epsilon):
            action = random.choice(legal_actions)
        else:
            action = self.computeActionFromQValues(state)

        return action

    def getAction(self, state):
        """
          Compute the action to take in the current state.  With
          probability self.epsilon, we should take a random action and
          take the best policy action otherwise.  Note that if there are
          no legal actions, which is the case at the terminal state, you
          should choose None as the action.

          HINT: You might want to use util.flipCoin(prob)
          HINT: To pick randomly from a list, use random.choice(list)
        """
        # Pick Action
        legalActions = self.getLegalActions(state)
        action = None
        "*** YOUR CODE HERE ***"
        if not self.qTable.has_key(state):
            
            self.qTable[state] = {}
            for action in legalActions:
                self.qTable[state][action] = 0
        if len(legalActions) == 0:
            return None
        coin = util.flipCoin(self.epsilon)
        if coin == True :
            action = random.choice(legalActions)
        else:
            v = -9999
            for act in legalActions:
                if self.qTable[state][act] > v:
                    v = self.qTable[state][act]
                    action = act
                

        return action

    def getAction(self, hitOrnot,position):
        """
          Compute the action to take in the current state.  With
          probability self.epsilon, we should take a random action and
          take the best policy action otherwise.  Note that if there are
          no legal actions, which is the case at the terminal state, you
          should choose None as the action.

          HINT: You might want to use util.flipCoin(prob)
          HINT: To pick randomly from a list, use random.choice(list)
        """
        # Pick Action
        targetlist=[]
        legalDistance=[]
        target=()
        legalDistance =self.getAllpossibleDistance(position) 
#        print"GA position is", position
        if legalDistance:
            if util.flipCoin(self.epsilon):
#                print "length",len(legalDistance)
                random_Distance = random.randint(0, len(legalDistance)-1)
                shoot_distance=legalDistance[random_Distance]
#                print "GA shoot_distance:",shoot_distance
                targetlist=self.findLocationWithShootDistance(position,shoot_distance)
#                print"GA TARGET LIST",targetlist,"len is",len(targetlist)
                randomTarget=random.randint(0, len(targetlist)-1)
                target=targetlist[randomTarget]
                print "shoot randomly at",target,self.q_shot_counter
            else:
                target = self.getPolicy(hitOrnot,position)
        return target

 def getAction(self, state):
   """
     What action to take in the current state. With
     probability self.epsilon, we should take a random
     action and take the best policy action otherwise.
   
     After you choose an action make sure to
     inform your parent self.doAction(state,action) 
     This is done for you, just don't clobber it
      
     HINT: you might want to use util.flipCoin
     here..... (see util.py)
   """  
   # Pick Action
   action = None
   epsilon = self.epsilon
   take_random_action = util.flipCoin(epsilon)
   list_of_actions = self.getLegalActions(state)
   if take_random_action:
       
       action = random.choice(list_of_actions)
   else:
       action = self.getPolicy(state)
   #return action
   # Need to inform parent of action for Pacman
   self.doAction(state,action)    
   return action

  def getAction(self, state):
    """
      Compute the action to take in the current state.  With
      probability self.epsilon, we should take a random action and
      take the best policy action otherwise.  Note that if there are
      no legal actions, which is the case at the terminal state, you
      should choose None as the action.

      HINT: You might want to use util.flipCoin(prob)
      HINT: To pick randomly from a list, use random.choice(list)
    """
    # Pick Action
    "*** YOUR CODE HERE ***"
    s = state
    legalActions = self.getLegalActions(state)
    #print 'LEGAL:'+str(legalActions)
    if len(legalActions)==0:
      #print 'NONE'
      return None

    action = None

    if util.flipCoin(self.epsilon):
      action = random.choice(legalActions)
    else:
      action = self.getPolicy(s)
      #print action
    #print 'return' +str(action)  
    return action

    def chooseAction(self, state):
        #return random.choice( state.getLegalActions( self.index ) )
        if not self.firstTurnComplete:
            self.registerInitialState(state)
            self.firstTurnComplete = True
        

        """
        Picks among the actions with the highest Q(s,a).
        """
        actions = state.getLegalActions(self.index)
    
        if util.flipCoin(self.explorationRate):
            return random.choice(actions)
        
        # You can profile your evaluation time by uncommenting these lines
        # start = time.time()
        values = [(a, self.evaluate(state, a)) for a in actions]
        # print 'eval time for agent %d: %.4f' % (self.index, time.time() - start)
    
    
        #print 'VALUES: ' + str(values)  
        maxValue = max(values, key=lambda val : val[1])
        bestActions = [a for a, v in zip(actions, values) if v == maxValue]
    
        action = random.choice(bestActions)
                
        self.update(state, action, self.getSuccessor(state, action))
      
                
        #print 'Features: ' + str(self.getFeatures)
        #print 'Weights: ' + str(self.weights)
        #print 'Action: ' + str(action) + ' - ' + str(self.getPosition(state)) + '--->' + str(self.getPosition(self.getSuccessor(state, action)))
        return action

    def getAction(self, state):
        """
          Compute the action to take in the current state.  With
          probability self.epsilon, we should take a random action and
          take the best policy action otherwise.  Note that if there are
          no legal actions, which is the case at the terminal state, you
          should choose None as the action.

          HINT: You might want to use util.flipCoin(prob)
          HINT: To pick randomly from a list, use random.choice(list)
        """
        # Pick Action
        legalActions = self.getLegalActions(state)
        action = None
        "*** YOUR CODE HERE ***"
        #Calculate probability for taking actionS
        if len(legalActions) > 0:
            #Using probability from self.epsilon
            if util.flipCoin( self.epsilon):
                #Get random action from list using random.choice
                action = random.choice( legalActions)
            else:
                #Get action from Policy pie.
                action = self.getPolicy( state)
        return action

 def getAction(self, state):
     # Acc to some probability, we take a random action
     # Otherwise, we follow the best action available
     if util.flipCoin(self.epsilon):
       return random.choice(self.getLegalActions(state))
     else:
       return self.getPolicy(state)

  def getAction(self, state):
    """
      Compute the action to take in the current state.  With
      probability self.epsilon, we should take a random action and
      take the best policy action otherwise.  Note that if there are
      no legal actions, which is the case at the terminal state, you
      should choose None as the action.

      HINT: You might want to use util.flipCoin(prob)
      HINT: To pick randomly from a list, use random.choice(list)
    """
    # Pick Action
    legalActions = self.getLegalActions(state)
    "*** YOUR CODE HERE ***"

    # terminal state
    if not legalActions:
      return None
    else:
      # pick if we should explore by flipping a coin
      goRandom = util.flipCoin(self.epsilon)
      if goRandom:
        # randomly choose an action
        return random.choice(legalActions)
      else:
        # choose the best action
        return self.getPolicy(state)

  def getAction(self, state):
    """
      Compute the action to take in the current state.  With
      probability self.epsilon, we should take a random action and
      take the best policy action otherwise.  Note that if there are
      no legal actions, which is the case at the terminal state, you
      should choose None as the action.

      HINT: You might want to use util.flipCoin(prob)
      HINT: To pick randomly from a list, use random.choice(list)
    """
    # Pick Action
    legalActions = self.getLegalActions(state)
    action = None

    """Description:
    If the flip of the coin is favorable it will chose a random action, else it will get the best
    """
    """ YOUR CODE HERE """
    if not legalActions:
      return None

    if util.flipCoin(self.epsilon):
      action = random.choice(legalActions)
    else:
      action = self.getPolicy(state)
    """ END CODE """

    return action

    def getAction(self, state):
        """
      Compute the action to take in the current state.  With
      probability self.epsilon, we should take a random action and
      take the best policy action otherwise.  Note that if there are
      no legal actions, which is the case at the terminal state, you
      should choose None as the action.

      HINT: You might want to use util.flipCoin(prob)
      HINT: To pick randomly from a list, use random.choice(list)
    """
        # Pick Action

        legalActions = self.getLegalActions(state)
        action = None
        "*** YOUR CODE HERE ***"
        if len(legalActions) == 0:
            return None
        else:
            prob = util.flipCoin(self.epsilon)
            if prob:
                return random.choice(legalActions)
            else:
                q = util.Counter()
                for a in legalActions:
                    """if self.getQValue(state, a) > result or action == None :
                    action = a
                    result = self.getQValue(state, a)"""
                    q[state, a] = self.getQValue(state, a)
                return q.argMax()[1]

  def getAction(self, state):
    """
      Compute the action to take in the current state.  With
      probability self.epsilon, we should take a random action and
      take the best policy action otherwise.  Note that if there are
      no legal actions, which is the case at the terminal state, you
      should choose None as the action.

      HINT: You might want to use util.flipCoin(prob)
      HINT: To pick randomly from a list, use random.choice(list)
    """
    actions = self.getLegalActions(state)
    bestQValue = -99999999
    bestActions = []
    for action in actions:
       q = self.getQValue(state, action)
       if q == bestQValue:
          bestActions.append(action)
       elif q > bestQValue:
          bestActions = [action]
          bestQValue = q
    if len(bestActions) == 0:
          return None

    # Pick Action
    legalActions = self.getLegalActions(state)
    action = None
    if legalActions:
        if util.flipCoin(self.epsilon):
            action = random.choice(legalActions)
        else:
            action = random.choice(bestActions)
    return action