def on_monitor(self, model, dataset, algorithm):
        """
        Make sure Polyak-averaged model gets monitored.
        Save the model if necessary.

        Parameters
        ----------
        model : a Model instance
        dataset : Dataset
        algorithm : WRITEME
        """
        if self._count == self.start:
            self._worker = _PolyakWorker(model)
            algorithm.update_callbacks.append(self._worker)
            #HACK
            try:
                model.add_polyak_channels(self._worker.param_to_mean,
                                          algorithm.monitoring_dataset)
            except AttributeError:
                pass
        elif self.save_path is not None and self._count > self.start and \
                self._count % self.save_freq == 0:
            saved_params = OrderedDict()
            for param in model.get_params():
                saved_params[param] = param.get_value()
                param.set_value(self._worker.param_to_mean[param].get_value())
            serial.save(self.save_path, model)
            for param in model.get_params():
                param.set_value(saved_params[param])
        self._count += 1

def main(train_path,
        out_path,
        dataset,
        standardize,
        C,
        **kwargs):

    stl10 = dataset == 'stl10'
    cifar10 = dataset == 'cifar10'
    cifar100 = dataset == 'cifar100'
    assert stl10 + cifar10 + cifar100 == 1

    print('getting labels and oflds')
    train_y, fold_indices = get_labels_and_fold_indices(cifar10, cifar100, stl10)
    gc.collect()
    assert train_y is not None

    print('loading training features')
    train_X = get_features(train_path, split = False, standardize = standardize)

    assert str(train_X.dtype) == 'float32'
    if stl10:
        assert train_X.shape[0] == 5000
    if cifar10 or cifar100:
        assert train_X.shape[0] == 50000
        assert train_y.shape == (50000,)

    print('training model')
    model =  train(train_X, train_y, C)

    print('saving model')
    serial.save(out_path, model)

def cnn_ensemble_leave_one_out():
    import os
    from datetime import datetime
    print str(datetime.now())
    t0 = time.time()
    which_fold = 0
    while 1:
        print which_fold
        try:
            kwargs = get_default_configure_leave_one_out(which_fold=which_fold)
        except:
            break
        kwargs['num_rows']=40
        kwargs['num_cols']=40
        pp=pprint.PrettyPrinter(indent=4)
        pp.pprint(kwargs)
        kwargs['save_path'] =  str(which_fold)+'.pkl'
        t1 = time.time()
        ann = cnn_train(**kwargs)
        serial.save(kwargs['predict_path']+'f'+kwargs['save_path'],ann,on_overwrite='backup')
        print 'saved to: '+kwargs['save_path']
        print 'Traing done. Take {}h'.format((time.time()-t1)/3600)
        which_fold += 1
    utils.sms_notice('Training finished. Taking {}h in total.'.format((time.time()-t0)/3600))
    print 'Traing done. Take {}h'.format((time.time()-t0)/3600)
    # sum of all predictions
    #predict_batch()
    evaluate_sets(kwargs['predict_path'])

def cnn_transform_ensemble():
    import os
    from datetime import datetime
    print str(datetime.now())
    t0 = time.time()
    kwargs = get_default_configure()
    kwargs['num_rows']=41
    kwargs['num_cols']=41
    import pprint
    pp=pprint.PrettyPrinter(indent=4)
    pp.pprint(kwargs)

    i = -1
    while under_sample_water.which_fold != i:
        if i == -1:
            i = under_sample_water.which_fold
        kwargs['save_path'] =  str(under_sample_water.which_fold)+'.pkl'
        t1 = time.time()
        ann = cnn_train_tranformer(**kwargs)
        serial.save(kwargs['predict_path']+'f'+kwargs['save_path'],ann,on_overwrite='backup')
        print 'saved to: '+kwargs['save_path']
        print 'Traing done. Take {}h'.format((time.time()-t1)/3600)
        break
    utils.sms_notice('Training finished. Taking {}h in total.'.format((time.time()-t0)/3600))
    print 'Traing done. Take {}h'.format((time.time()-t0)/3600)
    # sum of all predictions
    predict_batch(predict_path)

    def save_weights(self):
        """
        Saves all weights in a .txt, .npy or .mat file depending on the ending of the 'weight_path'.
        If the path ends in .pkl, the entire model is stored. 
        """
    
        model = serial.load(self.model_path)
    
        weight_dict = {}
    
        for layer in model.layers:
            try:
                weight_dict[layer.layer_name] = layer.get_weights()
            except:
                layer_weights = layer.get_weights_topo()
                weight_dict[layer.layer_name] = layer_weights# without reshaping since it the input/output vector would need to reshaped in the same way which might lead to problems

        if self.weight_path[-4:] == '.pkl':
            print 'saving model ', self.weight_path
            serial.save(self.weight_path, model)
        elif self.weight_path[-4:] == '.mat':
            scipy.io.savemat(self.weight_path[:-4]+'.mat', weight_dict)
        elif self.weight_path[-4:] == '.npy':
            np.save(self.weight_path[:-4], weight_dict)
        else:
            raise Exception('Only ".mat", ".pkl" and ".npy" files are supported as data formats.')

def get_processed_dataset():

    train_path = 'pp_cifar10_train.pkl'
    test_path = 'pp_cifar10_test.pkl'

    if os.path.exists(train_path) and os.path.exists(test_path):
        print 'loading preprocessed data'
        trainset = serial.load(train_path)
        testset = serial.load(test_path)

    else:
        print 'loading raw data...'
        trainset = cifar10.CIFAR10(which_set="train")
        testset =  cifar10.CIFAR10(which_set="test")
	
        pipeline = preprocessing.Pipeline()
        pipeline.items.append(preprocessing.ExtractPatchesWithPosition(patch_shape=patch_shape, patches_per_image=patches_per_image))
        pipeline.items.append(preprocessing.GlobalContrastNormalization(sqrt_bias=10., use_std=True))
        pipeline.items.append(preprocessing.PCA(num_components = num_components, keep_var_fraction = keep_var_fraction))
        pipeline.items.append(preprocessing.ExtractPatchPairs(patches_per_image = patches_per_image, num_images = train_size, input_width = input_width))

        trainset.apply_preprocessor(preprocessor=pipeline, can_fit=True)

        # the pkl-ing is having issues, the dataset is maybe too big.
        serial.save('pp_cifar10_train.pkl', trainset)
        serial.save('pp_cifar10_test.pkl', testset)

        # this path will be used for visualizing weights after training is done
        trainset.yaml_src = '!pkl: "%s"' % train_path
        testset.yaml_src = '!pkl: "%s"' % test_path

    return trainset, testset

 def on_monitor(self, model, dataset, algorithm):
     
     epoch = algorithm.monitor._epochs_seen;
     model_file = self.save_path + self.save_prefix + str(epoch) + '.pkl'; 
     
     with log_timing(log, 'saving model to {}'.format(model_file)):
         serial.save(model_file, model, on_overwrite = 'backup')

    def on_monitor(self, model, dataset, algorithm):
        """
        Looks whether the model performs better than earlier. If it's the
        case, saves the model.

        Parameters
        ----------
        model : pylearn2.models.model.Model
                model.monitor must contain a channel with name given by self.channel_name
        dataset : pylearn2.datasets.dataset.Dataset
            not used
        algorithm : TrainingAlgorithm
            not used
        """

        monitor = model.monitor
        channels = monitor.channels
        channel = channels[self.channel_name]
        val_record = channel.val_record
        new_cost = self.coeff * val_record[-1]

        if new_cost < self.best_cost:
            self.best_cost = new_cost
            serial.save(self.save_path, model, on_overwrite = 'backup')
            
            # XXX: [Kien] Save best filters.
            pv = get_weights_report.get_weights_report(model = model, 
                                                       dataset = dataset)
            pv.save('best_filters.png')                                           

    def on_monitor(self, model, dataset, algorithm):
        """
        Looks whether the model performs better than earlier
        - or equally good (modification).
        If it's the case, saves the model.

        Parameters
        ----------
        model : pylearn2.models.model.Model
            model.monitor must contain a channel with name given by
            self.channel_name
        dataset : pylearn2.datasets.dataset.Dataset
            Not used
        algorithm : TrainingAlgorithm
            Not used
        """
        monitor = model.monitor
        channels = monitor.channels
        channel = channels[self.channel_name]
        val_record = channel.val_record
        new_cost = val_record[-1]

        if self.coeff * new_cost <= self.coeff * self.best_cost and \
           monitor._epochs_seen >= self.start_epoch:
            self.best_cost = new_cost
            # Update the tag of the model object before saving it.
            self._update_tag(model)
            if self.store_best_model:
                self.best_model = deepcopy(model)
            if self.save_path is not None:
                with log_timing(log, 'Saving to ' + self.save_path):
                    serial.save(self.save_path, model, on_overwrite='backup')

    def train_batch(self, dataset, batch_size):

        x = dataset.get_batch_design(batch_size, include_labels=False)
        self.batch_train_func(x)

        # accounting...
        self.examples_seen += self.batch_size
        self.batches_seen += 1

        # modify learning rate multipliers
        for (k, iter) in self.lr_mults_it.iteritems():
            if iter.next():
                print 'self.batches_seen = ', self.batches_seen
                self.lr_mults_shrd[k].set_value(iter.value)
                print 'lr_mults_shrd[%s] = %f' % (k,iter.value)

        self.enforce_constraints()

        # save to different path each epoch
        if self.my_save_path and \
           (self.batches_seen in self.save_at or
            self.batches_seen % self.save_every == 0):
            fname = self.my_save_path + '_e%i.pkl' % self.batches_seen
            print 'Saving to %s ...' % fname,
            serial.save(fname, self)
            print 'done'

        return self.batches_seen < self.max_updates

    def save(self):
        """
        Call on_save for Train and TrainCV extensions and serialize trained
        models if save_path is set.
        """
        # Train extensions
        for trainer in self.trainers:
            for extension in trainer.extensions:
                extension.on_save(trainer.model, trainer.dataset,
                                  trainer.algorithm)

        # TrainCV extensions
        for extension in self.cv_extensions:
            extension.on_save(self.trainers)

        # serialize trained models
        if self.save_path is not None:
            models = [trainer.model for trainer in self.trainers]
            try:
                for trainer in self.trainers:
                    trainer.dataset._serialization_guard = SerializationGuard()
                if not self.allow_overwrite and os.path.exists(self.save_path):
                    raise IOError("Trying to overwrite file when not allowed.")
                serial.save(self.save_path, models, on_overwrite='backup')
            finally:
                for trainer in self.trainers:
                    trainer.dataset._serialization_guard = None

    def on_monitor(self, model, dataset, algorithm):
        """
        Save the model if we are on a save epoch.
        
        Parameters
        ----------
        model : pylearn2.models.model.Model
                model.monitor must contain a channel with name given by self.channel_name
        dataset : pylearn2.datasets.dataset.Dataset
            not used
        algorithm : TrainingAlgorithm
            not used
        """

        #monitor = model.monitor
        #channels = monitor.channels
        #channel = channels[self.channel_name]
        #val_record = channel.val_record
        #epoch = len(val_record)
        epoch = model.monitor.get_epochs_seen()

        save_file = '%s_%d.pkl' % (self.save_prefix, epoch)

        if np.mod(epoch, self.interval) == 0:
            print('Saving model to %s' % save_file)
            serial.save(save_file, model, on_overwrite = 'backup')

    def train_batch(self, dataset, batch_size):

        x = dataset.get_batch_design(batch_size, include_labels=False)
        if self.flags['truncate_v']:
            x = numpy.clip(x, -self.truncation_bound['v'], self.truncation_bound['v'])
        try:
            self.batch_train_func(x)
            self.enforce_constraints()
        except:
            import pdb; pdb.set_trace()

        # accounting...
        self.examples_seen += self.batch_size
        self.batches_seen += 1

        # save to different path each epoch
        if self.my_save_path and \
           (self.batches_seen in self.save_at or
            self.batches_seen % self.save_every == 0):
            fname = self.my_save_path + '_e%i.pkl' % self.batches_seen
            print 'Saving to %s ...' % fname,
            serial.save(fname, self)
            print 'done'

        return self.batches_seen < self.max_updates

    def on_monitor(self, model, dataset, algorithm):
        # this shall never happen but better safe than sorry
        if self.predictor is None:
            self.setup(model, dataset, algorithm)

        # obtaining validating set # TODO: finally we want to have train-validation-test set. Or sth.
        valid_x = algorithm.monitoring_dataset['valid'].X
        valid_y = algorithm.monitoring_dataset['valid'].y
        predictions = self.predictor.get_predictions(valid_x)
        threshold, score = self.compute_optimal_threshold_and_score(valid_y, predictions)

        self.threshold_list.append(threshold)
        self.score_list.append(score)

        if self.saving_path is not None and self.save:
            if max(self.score_list) == score:
                try:
                    # Make sure that saving does not serialize the dataset
                    dataset._serialization_guard = SerializationGuard()
                    save_path = self.saving_path
                    serial.save(save_path, model,
                                on_overwrite='backup')
                finally:
                    dataset._serialization_guard = None

        print "F1Score1Threshold score", score, "\ncorresponding threshold:", threshold

    def on_monitor(self, model, dataset, algorithm):
        import numpy as np
        # this shall never happen but better safe than sorry
        if self.predictor is None:
            self.setup(model, dataset, algorithm)

        # obtaining validating set #
        valid_x = algorithm.monitoring_dataset['valid'].X
        valid_y = algorithm.monitoring_dataset['valid'].y
        y_pred = self.predictor.get_predictions(valid_x)
        y_classes = [np.argmax(pred) for pred in y_pred]
        score = f1_score(y_true=valid_y, y_pred=y_classes)
        self.score_list.append(score)

        if self.saving_path is not None and self.save:
            if max(self.score_list) == score:
                try:
                    # Make sure that saving does not serialize the dataset
                    dataset._serialization_guard = SerializationGuard()
                    save_path = self.saving_path
                    serial.save(save_path, model,
                                on_overwrite='backup')
                finally:
                    dataset._serialization_guard = None

        print "F1 score:", score

def main(train_path,
        out_path,
        split,
        **kwargs):



    print 'loading training features'

    train_X = get_features(train_path, split)
    #assert train_X.flags.c_contiguous
    gc.collect()


    assert str(train_X.dtype) == 'float32'
    assert train_X.shape[0] == 120

    report = Report(train_path, split)

    train_X_omnivore, train_y, fold_indices = get_training_subset(train_X, 'omnivore')

    model = train(fold_indices, train_X_omnivore, train_y, report, **kwargs)

    serial.save(out_path+'.omnivore.model.pkl', model)
    report.write(out_path+'.omnivore.validation_report.txt')


    report = Report(train_path, split)

    train_X_fruit, train_y, fold_indices = get_training_subset(train_X, 'fruit')

    model = train(fold_indices, train_X_fruit, train_y, report, **kwargs)

    serial.save(out_path+'.fruit.model.pkl', model)
    report.write(out_path+'.fruit.validation_report.txt')

    def on_monitor(self, model, dataset, algorithm):
        """
        Looks whether the model performs better than earlier. If it's the
        case, saves the model.

        Parameters
        ----------
        model : pylearn2.models.model.Model
            model.monitor must contain a channel with name given by
            self.channel_name
        dataset : pylearn2.datasets.dataset.Dataset
            Not used
        algorithm : TrainingAlgorithm
            Not used
        """

        monitor = model.monitor
        channels = monitor.channels
        channel = channels[self.channel_name]
        val_record = channel.val_record
        new_cost = self.coeff * val_record[-1]


        if new_cost < self.best_cost:
            self.best_cost = new_cost
            # Update the tag of the model object before saving it.
            self._update_tag(model)
            serial.save(self.save_path, model, on_overwrite = 'backup')

    def on_save(self, trainers):
        """
        Save best model from each cross-validation fold.

        Parameters
        ----------
        trainers : list
            List of Train objects belonging to the parent TrainCV object.
        """
        if self.save_path is None:
            return
        models = []
        for trainer in trainers:
            for extension in trainer.extensions:
                if isinstance(extension, MonitorBasedSaveBest):
                    models.append(extension.best_model)
                    break
        assert len(models) == len(trainers)
        try:
            for trainer in trainers:
                trainer.dataset._serialization_guard = SerializationGuard()
                serial.save(self.save_path, models, on_overwrite='backup')
        finally:
            for trainer in trainers:
                trainer.dataset._serialization_guard = None

def compute_ZCA_fast(X, normalize, ZCA_filename="zca"):
    zca_preprocessor = preprocessing.ZCA()
    zca_preprocessor.set_matrices_save_path(ZCA_filename+".npz")
    X = X.astype(np.float32)
    if normalize:
        X /= 255.0
    zca_preprocessor.fit(X.T)
    serial.save(ZCA_filename+".pkl", zca_preprocessor)

 def save(self, filename):
     # Delete data sets
     if (hasattr(self.experiment.binary_csp, 'cnt')):
         del self.experiment.binary_csp.cnt
     if hasattr(self.experiment, 'test_cnt'):
         del self.experiment.test_cnt
     del self.experiment.cnt
     serial.save(filename, self.experiment)