def __init__(self):
     # standard parameters
     self.epsilon = 2.0 #Initial value of sigmas
     self.baseline=0.0 #Moving average baseline, used for sigma adaption
     self.best=-1000000.0 #TODO ersetzen durch -inf
     self.symCount=1.0 #Switch for symetric sampling
     self.gd = GradientDescent()
     self.gdSig = GradientDescent()
     self.wDecay = 0.001 #lasso weight decay (0 to deactivate)

class FiniteDifferences(ContinuousOptimizer):
    """ Basic finite difference method. """

    epsilon = 1.0
    gamma = 0.999
    batchSize = 10

    # gradient descent parameters
    learningRate = 0.1
    learningRateDecay = None
    momentum = 0.0
    rprop = False

    

    def _setInitEvaluable(self, evaluable):
        ContinuousOptimizer._setInitEvaluable(self, evaluable)
        self.current = self._initEvaluable
        self.gd = GradientDescent()
        self.gd.alpha = self.learningRate
        if self.learningRateDecay is not None:
            self.gd.alphadecay = self.learningRateDecay
        self.gd.momentum = self.momentum
        self.gd.rprop = self.rprop
        self.gd.init(self._initEvaluable)

    def perturbation(self):
        """ produce a parameter perturbation """
        deltas = random.uniform(-self.epsilon, self.epsilon, self.numParameters)
        # reduce epsilon by factor gamma
        self.epsilon *= self.gamma
        return deltas

    def _learnStep(self):
        """ calls the gradient calculation function and executes a step in direction
            of the gradient, scaled with a small learning rate alpha. """

        # initialize matrix D and vector R
        D = ones((self.batchSize, self.numParameters))
        R = zeros((self.batchSize, 1))

        # calculate the gradient with pseudo inverse
        for i in range(self.batchSize):
            deltas = self.perturbation()
            x = self.current + deltas
            D[i, :] = deltas
            R[i, :] = self._oneEvaluation(x)
        beta = dot(pinv(D), R)
        gradient = ravel(beta)

        # update the weights
        self.current = self.gd(gradient)

class FDBasic(FDLearner):
    def __init__(self):
        # standard parameters
        self.epsilon = 1.0
        self.gamma = 0.999
        self.gd = GradientDescent()

    def setModule(self, module):
        FDLearner.setModule(self, module)        
        self.gd.init(self.original)

    def perturbate(self):
        """ perturb the parameters. """
        # perturb the parameters and store the deltas in dataset
        deltas = random.uniform(-self.epsilon, self.epsilon, self.module.paramdim)
        # reduce epsilon by factor gamma
        self.epsilon *= self.gamma
        self.ds.append('deltas', deltas)
        # change the parameters in module (params is a pointer!)
        params = self.module.params
        params[:] = self.original + deltas

    def learn(self):
        """ calls the gradient calculation function and executes a step in direction
            of the gradient, scaled with a small learning rate alpha. """
        assert self.ds != None
        assert self.module != None
        
        # get the deltas from the dataset
        deltas = self.ds.getField('deltas')
        
        # initialize matrix D and vector R
        D = ones((self.ds.getNumSequences(), self.module.paramdim + 1))
        R = zeros((self.ds.getNumSequences(), 1))
        
        # calculate the gradient with pseudo inverse
        for seq in range(self.ds.getNumSequences()):
            _state, _action, reward = self.ds.getSequence(seq)
            D[seq,:-1] = deltas[seq,:]
            R[seq,:] = mean(reward)
        
        beta = dot(pinv(D), R)        
        gradient = ravel(beta[:-1])
        
        # update the weights
        self.original = self.gd(gradient)       
        self.module._setParameters(self.original)
           
        self.module.reset()

 def __init__(self, module, dataset=None, learningrate=0.01, lrdecay=1.0,
              momentum=0., verbose=False, batchlearning=False,
              weightdecay=0.):
     """Create a BackpropTrainer to train the specified `module` on the
     specified `dataset`.
     The learning rate gives the ratio of which parameters are changed into
     the direction of the gradient. The learning rate decreases by `lrdecay`,
     which is used to to multiply the learning rate after each training
     step. The parameters are also adjusted with respect to `momentum`, which
     is the ratio by which the gradient of the last timestep is used.
     If `batchlearning` is set, the parameters are updated only at the end of
     each epoch. Default is False.
     `weightdecay` corresponds to the weightdecay rate, where 0 is no weight
     decay at all.
     """
     Trainer.__init__(self, module)
     self.setData(dataset)
     self.verbose = verbose
     self.batchlearning = batchlearning
     self.weightdecay = weightdecay
     self.epoch = 0
     self.totalepochs = 0
     # set up gradient descender
     self.descent = GradientDescent()
     self.descent.alpha = learningrate
     self.descent.momentum = momentum
     self.descent.alphadecay = lrdecay
     self.descent.init(module.params)

 def __init__(self):
     # standard parameters
     self.epsilon = 2.0 #Initial value of exploration size
     self.baseline=0.0 #Moving average baseline, used just for visualisation
     self.best=-1000000.0 #TODO ersetzen durch -inf
     self.symCount=1.0 #Switch for symetric sampling
     self.gd = GradientDescent()
     self.gamma=0.9995 #Exploration decay factor

 def _additionalInit(self):
     if self.sigmaLearningRate is None:
         self.sigmaLearningRate = self.learningRate    
     self.gdSig = GradientDescent()
     self.gdSig.alpha = self.sigmaLearningRate
     self.gdSig.rprop = self.rprop
     self.sigList = ones(self.numParameters) * self.epsilon #Stores the list of standard deviations (sigmas)
     self.gdSig.init(self.sigList)
     self.baseline = None

 def _setInitEvaluable(self, evaluable):
     ContinuousOptimizer._setInitEvaluable(self, evaluable)
     self.current = self._initEvaluable
     self.gd = GradientDescent()
     self.gd.alpha = self.learningRate
     if self.learningRateDecay is not None:
         self.gd.alphadecay = self.learningRateDecay
     self.gd.momentum = self.momentum
     self.gd.rprop = self.rprop
     self.gd.init(self._initEvaluable)

class PolicyGradientLearner(RLLearner):
    """ The PolicyGradientLearner takes a ReinforcementDataSet which has been extended with the log likelihood
        of each parameter for each time step. It additionally takes a Module which has been extended with
        a gaussian layer on top. It then changes the weights of both the gaussian layer and the rest of the
        module according to a specific gradient descent, which is a derivation of this base class.
    """
    def __init__(self):
        self.gd = GradientDescent()
        
    def setAlpha(self, alpha):
        """ pass the alpha value through to the gradient descent object """
        self.gd.alpha = alpha
        print "patching through to gd", alpha
    
    def getAlpha(self):
        return self.gd.alpha
    
    alpha = property(getAlpha, setAlpha)
        
    def setModule(self, module):
        RLLearner.setModule(self, module)
        self.gd.init(module.params)    
        
    def learn(self):
        """ calls the gradient calculation function and executes a step in direction
            of the gradient, scaled with a small learning rate alpha. """
        assert self.ds != None
        assert self.module != None
        
        # calculate the gradient with the specific function from subclass
        gradient = ravel(self.calculateGradient())

        # scale gradient if it has too large values
        if max(gradient) > 1000:
            gradient = gradient / max(gradient) * 1000
        
        # update the parameters
        p = self.gd(gradient)
        self.module._setParameters(p)
        self.module.reset()
        
    def calculateGradient(self):
        abstractMethod()

 def __init__(self):        
     # gradient descender
     self.gd = GradientDescent()
     
     # create default explorer
     self._explorer = None
     
     # loglh dataset
     self.loglh = None
     
     # network to tie module and explorer together
     self.network = None

 def __init__(self, module, dataset=None, learningrate=0.01, lrdecay=1.0,
              momentum=0., verbose=False, batchlearning=False,
              weightdecay=0.):
     Trainer.__init__(self, module)
     self.setData(dataset)
     self.verbose = verbose
     self.batchlearning = batchlearning
     self.weightdecay = weightdecay
     self.epoch = 0
     self.totalepochs = 0
     # set up gradient descender
     self.descent = GradientDescent()
     self.descent.alpha = learningrate
     self.descent.momentum = momentum
     self.descent.alphadecay = lrdecay
     self.descent.init(module.params)

    def __init__(self, module, dataset=None, learningrate=0.01, lrdecay=1.0,
                 momentum=0., verbose=False, batchlearning=False,
                 weightdecay=0., errfun=None):
        """Create a BackpropTrainer to train the specified `module` on the
        specified `dataset`.

        The learning rate gives the ratio of which parameters are changed into
        the direction of the gradient. The learning rate decreases by
        `lrdecay`, which is used to to multiply the learning rate after each
        training step. The parameters are also adjusted with respect to
        `momentum`, which is the ratio by which the gradient of the last
        timestep is used.

        If `batchlearning` is set, the parameters are updated only at the end
        of each epoch. Default is False.

        `weightdecay` corresponds to the weightdecay rate, where 0 is no weight
        decay at all.

        Arguments:
            errfun (func): Function that takes 2 positional arguments,
                the target (true) and predicted (estimated) output vectors, and
                returns an estimate of the signed distance to the target (true)
                output. default = lambda targ, est: (targ - est))
        """
        Trainer.__init__(self, module)
        self.setData(dataset)
        self.verbose = verbose
        self.batchlearning = batchlearning
        self.weightdecay = weightdecay
        self.epoch = 0
        self.totalepochs = 0
        # set up gradient descender
        self.descent = GradientDescent()
        self.descent.alpha = learningrate
        self.descent.momentum = momentum
        self.descent.alphadecay = lrdecay
        self.descent.init(module.params)
        self.errfun = errfun or abs_error

class TTrainer(BackpropTrainer):
    def __init__(self, module, dataset=None, learningrate=0.01, lrdecay=1.0,
                 momentum=0., verbose=False, batchlearning=False,
                 weightdecay=0.):
        super(BackpropTrainer, self).__init__(module)
        #self.setData(dataset)
        self.verbose = False
        self.batchlearning = batchlearning
        self.weightdecay = weightdecay
        self.epoch = 0
        self.totalepochs = 0
        # set up gradient descender
        self.descent = GradientDescent()
        self.descent.alpha = learningrate
        self.descent.momentum = momentum
        self.descent.alphadecay = lrdecay
        self.descent.init(module.params)


    def minibatch_training(self, dataset):

        for i in xrange(50):
            #ds = dataset.splitWithProportion(0.01)[0]
            ds = dataset.batches(1000)
            self.trainOnDataset(ds)
            ds.clear()

    def trainUntilConvergence(self, dataset=None, maxEpochs=None, verbose=None,
                              continueEpochs=10, validationProportion=0.25,FOLDER = os.curdir, modelfile = 'network.model', file = None):
        """Train the module on the dataset until it converges.

        Return the module with the parameters that gave the minimal validation
        error.

        If no dataset is given, the dataset passed during Trainer
        initialization is used. validationProportion is the ratio of the dataset
        that is used for the validation dataset.

        If maxEpochs is given, at most that many epochs
        are trained. Each time validation error hits a minimum, try for
        continueEpochs epochs to find a better one."""
        epochs = 0
        if dataset == None:
            dataset = self.ds
        if verbose == None:
            verbose = self.verbose
        # Split the dataset randomly: validationProportion of the samples for
        # validation.
        trainingData, validationData = (
            dataset.splitWithProportion(1 - validationProportion))
        if not (len(trainingData) > 0 and len(validationData)):
            raise ValueError("Provided dataset too small to be split into training " +
                             "and validation sets with proportion " + str(validationProportion))
        self.ds = trainingData
        bestweights = self.module.params.copy()
        bestverr = self.testOnData(validationData)
        trainingErrors = []
        validationErrors = [bestverr]
        while True:
            if maxEpochs != None:
                print float(epochs) / maxEpochs
            trainErr = self.train()
            validErr = self.testOnData(validationData)


            errs = [trainErr,validErr]
            print errs
            if file != None:
                with open (os.path.join(FOLDER, file), 'a') as neurons:
                    neurons.write(str(errs))
                    neurons.write('\n')

            trainingErrors.append(trainErr)
            validationErrors.append(validErr)

            if epochs == 0 or validationErrors[-1] < bestverr:
                # one update is always done
                bestverr = validationErrors[-1]
                bestweights = self.module.params.copy()
                NetworkWriter.writeToFile(self.module, os.path.join(FOLDER, modelfile))
                #with open (os.path.join(FOLDER, modelfile), 'wb') as fileObject:
                #    pickle.dump(self.module,fileObject)

            if maxEpochs != None and epochs >= maxEpochs:
                self.module.params[:] = bestweights
                break
            epochs += 1

            if len(validationErrors) >= continueEpochs * 2:
                # have the validation errors started going up again?
                # compare the average of the last few to the previous few
                old = validationErrors[-continueEpochs * 2:-continueEpochs]
                new = validationErrors[-continueEpochs:]
                if min(new) > max(old):
                    self.module.params[:] = bestweights
                    break
        trainingErrors.append(self.testOnData(trainingData))
        self.ds = dataset

        return trainingErrors, validationErrors

class ExtendedBackpropTrainer(Trainer):
    """Trainer that trains the parameters of a module according to a
    supervised dataset (potentially sequential) by backpropagating the errors
    (through time)."""

    def __init__(self, module, dataset=None, learningrate=0.01, lrdecay=1.0,
                 momentum=0., verbose=False, batchlearning=False,
                 weightdecay=0.):
        """Create a BackpropTrainer to train the specified `module` on the
        specified `dataset`.
        The learning rate gives the ratio of which parameters are changed into
        the direction of the gradient. The learning rate decreases by `lrdecay`,
        which is used to to multiply the learning rate after each training
        step. The parameters are also adjusted with respect to `momentum`, which
        is the ratio by which the gradient of the last timestep is used.
        If `batchlearning` is set, the parameters are updated only at the end of
        each epoch. Default is False.
        `weightdecay` corresponds to the weightdecay rate, where 0 is no weight
        decay at all.
        """
        Trainer.__init__(self, module)
        self.setData(dataset)
        self.verbose = verbose
        self.batchlearning = batchlearning
        self.weightdecay = weightdecay
        self.epoch = 0
        self.totalepochs = 0
        # set up gradient descender
        self.descent = GradientDescent()
        self.descent.alpha = learningrate
        self.descent.momentum = momentum
        self.descent.alphadecay = lrdecay
        self.descent.init(module.params)

    def train(self):
        """Train the associated module for one epoch."""
        assert len(self.ds) > 0, "Dataset cannot be empty."
        self.module.resetDerivatives()
        errors = 0
        ponderation = 0.
        shuffledSequences = []
        for seq in self.ds._provideSequences():
            shuffledSequences.append(seq)
        shuffle(shuffledSequences)
        for seq in shuffledSequences:
            e, p = self._calcDerivs(seq)
            errors += e
            ponderation += p
            if not self.batchlearning:
                gradient = self.module.derivs - self.weightdecay * self.module.params
                new = self.descent(gradient, errors)
                if new is not None:
                    self.module.params[:] = new
                self.module.resetDerivatives()

        if self.verbose:
            print("Total error: {z: .12g}".format(z=errors / ponderation))
        if self.batchlearning:
            self.module._setParameters(self.descent(self.module.derivs))
        self.epoch += 1
        self.totalepochs += 1
        return errors / ponderation


    def _calcDerivs(self, seq):
        """Calculate error function and backpropagate output errors to yield
        the gradient."""
        self.module.reset()
        for sample in seq:
            self.module.activate(sample[0])
        error = 0
        ponderation = 0.
        for offset, sample in reversed(list(enumerate(seq))):
            # need to make a distinction here between datasets containing
            # importance, and others
            target = sample[1]
            outerr = target - self.module.outputbuffer[offset]
            if len(sample) > 2:
                importance = sample[2]
                error += 0.5 * dot(importance, outerr ** 2)
                ponderation += sum(importance)
                self.module.backActivate(outerr * importance)
            else:
                error += 0.5 * sum(outerr ** 2)
                ponderation += len(target)
                # FIXME: the next line keeps arac from producing NaNs. I don't
                # know why that is, but somehow the __str__ method of the
                # ndarray class fixes something,
                str(outerr)
                self.module.backActivate(outerr)

        return error, ponderation

    def _checkGradient(self, dataset=None, silent=False):
        """Numeric check of the computed gradient for debugging purposes."""
        if dataset:
            self.setData(dataset)
        res = []
        for seq in self.ds._provideSequences():
            self.module.resetDerivatives()
#.........这里部分代码省略.........

 def __init__(self):
     # standard parameters
     self.epsilon = 1.0
     self.gamma = 0.999
     self.gd = GradientDescent()

class SPLA(FDLearner):
    def __init__(self):
        # standard parameters
        self.epsilon = 2.0 #Initial value of sigmas
        self.baseline=0.0 #Moving average baseline, used for sigma adaption
        self.best=-1000000.0 #TODO ersetzen durch -inf
        self.symCount=1.0 #Switch for symetric sampling
        self.gd = GradientDescent()
        self.gdSig = GradientDescent()
        self.wDecay = 0.001 #lasso weight decay (0 to deactivate)

    def setModule(self, module):
        """Sets and initializes all module settings"""
        FDLearner.setModule(self, module)
        self.original = zeros(self.module.params.shape) #Stores the parameter set
        self.sigList=ones(self.module.params.shape) #Stores the list of standard deviations (sigmas)
        self.initSigmas()
        self.deltas=zeros(self.module.params.shape) #the parameter difference vector for exploration
        self.module._setParameters(self.original) #initializes the module parameter set to zeros
        self.gd.init(self.original)
        self.numOParas=len(self.original)

    def initSigmas(self):
        self.sigList*=self.epsilon #initialize sigmas to epsilon
        self.gdSig.init(self.sigList)
        
    def genDifVect(self):
        # generates a difference vector with the given standard deviations
        self.deltas=random.normal(0.0, self.sigList)
        
    def perturbate(self):
        """ perturb the parameters. """
        self.symCount*=-1.0 #change sign of perturbation
        self.ds.append('deltas', self.symCount*self.deltas) #add perturbation to the data set
        self.module._setParameters(self.original + self.symCount*self.deltas) #set the actual perturbed parameters in module

    def learn(self):
        """ calculates the gradient and executes a step in the direction
            of the gradient, scaled with a learning rate alpha. """
        assert self.ds != None
        assert self.module != None
                       
        # calculate the gradient
        reward1=0.0 #reward of positive perturbation
        reward2=0.0 #reward of negative perturbation
        sym=1.0 #perturbation switch
        seqLen=self.ds.getNumSequences() #number of sequences done for learning
        for seq in range(seqLen):
            sym*=-1.0
            _, _, reward = self.ds.getSequence(seq)
            #add up the rewards of positive and negative perturbation role outs respectivly
            if sym==1.0: reward1+=sum(reward)
            else: reward2+=sum(reward)
        #normate rewards by seqLen 
        reward1/=float(seqLen)
        reward2/=float(seqLen)
        self.reward=(reward1+reward2)
        reward1*=2.0
        reward2*=2.0

        #check if reward is the best observed up to now
        if reward1 > self.best: self.best= reward1
        if reward2 > self.best: self.best= reward2

        #some checks at the first learnign sequence
        if self.baseline==0.0: 
            self.baseline=self.reward/2.0
            fakt=0.0
            fakt2=0.0
            if seqLen/2 != float(seqLen)/2.0: 
                print "ATTENTON!!! SPLA uses symetric sampling! Number of episodes per learning step must be even! (2 for deterministic settings, >2 for stochastic settings) A numer of episodes of ", seqLen, "is odd."
                while(True): sleep(1)
        else: 
            #calc the gradients
            if reward1!=reward2:
                #gradient estimate alla SPSA but with liklihood gradient and normalization
                fakt=(reward1-reward2)/(2.0*self.best-reward1-reward2) 
            else: fakt=0.0
            #normalized sigma gradient with moving average baseline
            fakt2=(self.reward-self.baseline)/(self.best-self.baseline) 
        self.baseline=0.9*self.baseline+0.1*self.reward #update baseline
            
        # update parameters and sigmas
        self.original = self.gd(fakt*self.deltas-self.original*self.sigList*self.wDecay)   
        print abs(self.original).sum()/self.numOParas            
        if fakt2> 0.0: #for sigma adaption alg. follows only positive gradients
            self.sigList=self.gdSig(fakt2*(self.deltas*self.deltas-self.sigList*self.sigList)/self.sigList) #apply sigma update
        self.module.reset() # reset the module 
        self.genDifVect() #generate a new perturbation vector

class BackpropTrainerWiener(Trainer):

    def __init__(self, module, dataset=None, learningrate=0.01, lrdecay=1.0,
                 momentum=0., verbose=False, batchlearning=False,
                 weightdecay=0.):
        Trainer.__init__(self, module)
        self.setData(dataset)
        self.verbose = verbose
        self.batchlearning = batchlearning
        self.weightdecay = weightdecay
        self.epoch = 0
        self.totalepochs = 0
        # set up gradient descender
        self.descent = GradientDescent()
        self.descent.alpha = learningrate
        self.descent.momentum = momentum
        self.descent.alphadecay = lrdecay
        self.descent.init(module.params)

    def train(self):
        assert len(self.ds) > 0, "Dataset cannot be empty."
        self.module.resetDerivatives()
        errors = 0
        ponderation = 0.
        shuffledSequences = []
        for seq in self.ds._provideSequences():
            shuffledSequences.append(seq)
        shuffle(shuffledSequences)
        for seq in shuffledSequences:
            e, p = self._calcDerivs(seq)
            errors += e
            ponderation += p
            if not self.batchlearning:
                gradient = self.module.derivs - self.weightdecay * self.module.params
                new = self.descent(gradient, errors)
                if new is not None:
                    self.module.params[:] = new
                self.module.resetDerivatives()

        if self.verbose:
            print "Total error:", errors / ponderation
        if self.batchlearning:
            self.module._setParameters(self.descent(self.module.derivs))
        self.epoch += 1
        self.totalepochs += 1
        return errors / ponderation


    def _calcDerivs(self, seq):
        self.module.reset()
        for sample in seq:
            self.module.activate(sample[0])
        error = 0
        ponderation = 0.
        for offset, sample in reversed(list(enumerate(seq))):
            target = sample[1]
            outerr = target - self.module.outputbuffer[offset]
            if len(sample) > 2:
                importance = sample[2]
                error += 0.5 * dot(importance, outerr ** 2)
                ponderation += sum(importance)
                self.module.backActivate(outerr * importance)
            else:
                error += 0.5 * sum(outerr ** 2)
                ponderation += len(target)
                str(outerr)
                self.module.backActivate(outerr)

        return error, ponderation

    def _checkGradient(self, dataset=None, silent=False):
        if dataset:
            self.setData(dataset)
        res = []
        for seq in self.ds._provideSequences():
            self.module.resetDerivatives()
            self._calcDerivs(seq)
            e = 1e-6
            analyticalDerivs = self.module.derivs.copy()
            numericalDerivs = []
            for p in range(self.module.paramdim):
                storedoldval = self.module.params[p]
                self.module.params[p] += e
                righterror, dummy = self._calcDerivs(seq)
                self.module.params[p] -= 2 * e
                lefterror, dummy = self._calcDerivs(seq)
                approxderiv = (righterror - lefterror) / (2 * e)
                self.module.params[p] = storedoldval
                numericalDerivs.append(approxderiv)
            r = zip(analyticalDerivs, numericalDerivs)
            res.append(r)
            if not silent:
                print r
        return res

    def testOnData(self, dataset=None, verbose=False):
        if dataset == None:
            dataset = self.ds
        dataset.reset()
        if verbose:
#.........这里部分代码省略.........

class SimpleSPSA(FDLearner):
    def __init__(self):
        # standard parameters
        self.epsilon = 2.0 #Initial value of exploration size
        self.baseline=0.0 #Moving average baseline, used just for visualisation
        self.best=-1000000.0 #TODO ersetzen durch -inf
        self.symCount=1.0 #Switch for symetric sampling
        self.gd = GradientDescent()
        self.gamma=0.9995 #Exploration decay factor

    def setModule(self, module):
        """Sets and initializes all module settings"""
        FDLearner.setModule(self, module)
        self.original = zeros(self.module.params.shape) #Stores the parameter set
        self.module._setParameters(self.original) #initializes the module parameter set to zeros
        self.gd.init(self.original)
        self.numOParas=len(self.original)
        self.genDifVect()

    def genDifVect(self):
        # generates a uniform difference vector with the given epsilon
        self.deltas = (random.randint(0,2,self.numOParas)*2-1)*self.epsilon
        
    def perturbate(self):
        """ perturb the parameters. """
        self.symCount*=-1.0 #change sign of perturbation
        self.ds.append('deltas', self.symCount*self.deltas) #add perturbation to the data set
        self.module._setParameters(self.original + self.symCount*self.deltas) #set the actual perturbed parameters in module

    def learn(self):
        """ calculates the gradient and executes a step in the direction
            of the gradient, scaled with a learning rate alpha. """
        assert self.ds != None
        assert self.module != None
               
        # calculate the gradient
        reward1=0.0 #reward of positive perturbation
        reward2=0.0 #reward of negative perturbation
        sym=1.0 #perturbation switch
        seqLen=self.ds.getNumSequences() #number of sequences done for learning
        for seq in range(seqLen):
            sym*=-1.0
            _state, _action, reward = self.ds.getSequence(seq)
            #add up the rewards of positive and negative perturbation role outs respectivly
            if sym==1.0: reward1+=sum(reward)
            else: reward2+=sum(reward)
        #normate rewards by seqLen 
        reward1/=float(seqLen)
        reward2/=float(seqLen)
        self.reward=(reward1+reward2)
        reward1*=2.0
        reward2*=2.0

        #check if reward is the best observed up to now
        if reward1 > self.best: self.best= reward1
        if reward2 > self.best: self.best= reward2

        #some checks at the first learnign sequence
        if self.baseline==0.0: 
            self.baseline=self.reward*0.99
            fakt=0.0
            if seqLen/2 != float(seqLen)/2.0: 
                print "ATTENTON!!! SPSA uses symetric sampling! Number of episodes per learning step must be even! (2 for deterministic settings, >2 for stochastic settings) A numer of episodes of ", seqLen, "is odd."
                while(True): sleep(1)
        else: 
            #calc the gradients
            if reward1!=reward2:
                #gradient estimate alla SPSA but with liklihood gradient and normalization (see also "update parameters")
                fakt=(reward1-reward2)/(2.0*self.best-reward1-reward2) 
            else: fakt=0.0
        self.baseline=0.9*self.baseline+0.1*self.reward #update baseline
            
        # update parameters
        # as a simplification we use alpha = alpha * epsilon**2 for decaying the stepsize instead of the usual use method from SPSA
        # resulting in the same update rule like for PGPE
        self.original = self.gd(fakt*self.epsilon*self.epsilon/self.deltas)
        # reduce epsilon by factor gamma
        # as another simplification we let the exploration just decay with gamma. 
        # Is similar to the decreasing exploration in SPSA but simpler.
        self.epsilon *= self.gamma
        self.module.reset() # reset the module 
        self.genDifVect() #generate a new perturbation vector

 def __init__(self):
     self.gd = GradientDescent()

class PGPE(FiniteDifferences):
    """ Policy Gradients with Parameter Exploration (ICANN 2008)."""
    
    #:exploration type
    exploration = "local"
    #: specific settings for sigma updates
    learningRate = 0.2    
    #: specific settings for sigma updates
    sigmaLearningRate = 0.1
    #: Initial value of sigmas
    epsilon = 2.0
    #:lasso weight decay (0 to deactivate)
    wDecay = 0.0
    #:momentum term (0 to deactivate)
    momentum = 0.0
    #:rprop decent (False to deactivate)
    rprop = False
    
    def _additionalInit(self):
        if self.sigmaLearningRate is None:
            self.sigmaLearningRate = self.learningRate    
        self.gdSig = GradientDescent()
        self.gdSig.alpha = self.sigmaLearningRate
        self.gdSig.rprop = self.rprop
        self.sigList = ones(self.numParameters) * self.epsilon #Stores the list of standard deviations (sigmas)
        self.gdSig.init(self.sigList)
        self.baseline = None
        
    def perturbation(self):
        """ Generate a difference vector with the given standard deviations """
        return random.normal(0., self.sigList)
            
    def _learnStep(self):
        """ calculates the gradient and executes a step in the direction
            of the gradient, scaled with a learning rate alpha. """
        deltas = self.perturbation()
        #reward of positive and negative perturbations
        reward1 = self._oneEvaluation(self.current + deltas)        
        reward2 = self._oneEvaluation(self.current - deltas)

        self.mreward = (reward1 + reward2) / 2.                
        if self.baseline is None: 
            # first learning step
            self.baseline = self.mreward
            fakt = 0.
            fakt2 = 0.          
        else: 
            #calc the gradients
            if reward1 != reward2:
                #gradient estimate alla SPSA but with likelihood gradient and normalization
                fakt = (reward1 - reward2) / (2. * self.bestEvaluation - reward1 - reward2) 
            else: 
                fakt=0.
            #normalized sigma gradient with moving average baseline
            norm = (self.bestEvaluation-self.baseline)
            if norm != 0.0:
                fakt2=(self.mreward-self.baseline)/(self.bestEvaluation-self.baseline)
            else:
                fakt2 = 0.0
        #update baseline        
        self.baseline = 0.9 * self.baseline + 0.1 * self.mreward             
        # update parameters and sigmas
        self.current = self.gd(fakt * deltas - self.current * self.sigList * self.wDecay)   
        if fakt2 > 0.: #for sigma adaption alg. follows only positive gradients
            if self.exploration == "global":         
                #apply sigma update globally        
                self.sigList = self.gdSig(fakt2 * ((self.deltas ** 2).sum() - (self.sigList ** 2).sum())
                                          / (self.sigList * float(self.numParameters)))
            elif self.exploration == "local":
                #apply sigma update locally
                self.sigList = self.gdSig(fakt2 * (deltas * deltas - self.sigList * self.sigList) / self.sigList) 
            elif self.exploration == "cma":
                #I have to think about that - needs also an option in perturbation
                raise NotImplementedError()
            else:
                raise NotImplementedError(str(self.exploration) + " not a known exploration parameter setting.")

class PolicyGradientLearner(DirectSearchLearner, DataSetLearner, ExploringLearner):
    """ PolicyGradientLearner is a super class for all continuous direct search
        algorithms that use the log likelihood of the executed action to update
        the weights. Subclasses are ENAC, GPOMDP, or REINFORCE.
    """
    
    _module = None
    
    def __init__(self):        
        # gradient descender
        self.gd = GradientDescent()
        
        # create default explorer
        self._explorer = None
        
        # loglh dataset
        self.loglh = None
        
        # network to tie module and explorer together
        self.network = None
        
    
    def _setLearningRate(self, alpha):
        """ pass the alpha value through to the gradient descent object """
        self.gd.alpha = alpha
    
    def _getLearningRate(self):
        return self.gd.alpha
    
    learningRate = property(_getLearningRate, _setLearningRate)