def getFeatures(indexes):
            """
            .. todo::
                Write me
            """
            if self._load_to_memory:
                sequences = [self.samples_sequences[i] for i in indexes]
            else:
                sequences = [self.node[i] for i in indexes]
            # Get random source word index for "ngram"
            source_i = [numpy.random.randint(self.frame_length/2 +1, len(s)-self.frame_length/2, 1)[0] 
                        for s in sequences]
            target_i = [min(abs(int(numpy.random.normal(s_i, self.frame_length/3.0))), len(s)-1)
                        for s_i, s in safe_izip(source_i, sequences)]
            

            # Words mapped to integers greater than input max are set to 1 (unknown)
            X = [numpy.asarray([s[i]]) for i, s in safe_izip(source_i, sequences)]
            X[X>=self.X_labels] = numpy.asarray([1])
            X = numpy.asarray(X)
            y = [numpy.asarray([s[i]]) for i, s in safe_izip(target_i, sequences)]
            y[y>=self.X_labels] = numpy.asarray([1])
            y = numpy.asarray(y)
            # Store the targets generated by these indices.
            self.lastY = (y, indexes)
            #print X
            #print y
            return X

 def _fallback_next(self, next_index):
     # TODO: handle fancy-index copies by allocating a buffer and
     # using np.take()
     return tuple(
         fn(data[next_index]) if fn else data[next_index]
         for data, fn in safe_izip(self._raw_data, self._convert)
     )

    def iterator(self, mode=None, batch_size=None, num_batches=None,
                 rng=None, data_specs=None, return_tuple=False):
        allowed_modes = ('sequential', 'random_slice', 'even_sequential',
                         'batchwise_shuffled_sequential',
                         'even_batchwise_shuffled_sequential')
        if mode is not None and mode not in allowed_modes:
            raise ValueError("Due to HDF5 limitations on advanced indexing, " +
                             "the '" + mode + "' iteration mode is not " +
                             "supported")

        if data_specs is None:
            data_specs = self._iter_data_specs

        space, source = data_specs
        sub_spaces, sub_sources = (
            (space.components, source) if isinstance(space, CompositeSpace)
            else ((space,), (source,)))
        convert = [None for sp, src in safe_izip(sub_spaces, sub_sources)]

        mode = (self._iter_subset_class if mode is None
                else resolve_iterator_class(mode))

        if batch_size is None:
            batch_size = getattr(self, '_iter_batch_size', None)
        if num_batches is None:
            num_batches = getattr(self, '_iter_num_batches', None)
        if rng is None and mode.stochastic:
            rng = self.rng
        return VariableImageDatasetIterator(
            dataset=self,
            subset_iterator=mode(
                self.num_examples, batch_size, num_batches, rng),
            data_specs=data_specs,
            return_tuple=return_tuple,
            convert=convert)

    def next(self):
        next_index = self._subset_iterator.next()
        rvals = []
        if hasattr(self._dataset, 'get'):
            raw_data = self._next(next_index)
        else:
            raw_data = self._fallback_next(next_index)
        for (space, source, data, fn) in safe_izip(self._space, self._source,
                                                   #self._raw_data,
                                                   raw_data,
                                                   self._convert):
            rval = data
            if isinstance(space, SequenceDataSpace):
                # Add padding
                max_sequence_length = max(len(sample) for sample
                                          in data)
                batch = np.zeros((len(rval), max_sequence_length,
                                  space.dim), dtype=space.dtype)
                for i, sample in enumerate(rval):
                    batch[i, :len(sample)] = sample
                rvals.append(np.transpose(batch, (1, 0, 2)))

                # Create mask
                rvals.append(self._create_mask(rval))
            else:
                rvals.append(rval)

        # Reorder according to given data specs

        if not self._return_tuple and len(rval) == 1:
            rvals, = rvals
        return tuple(rvals)

    def __call__(self, model, X, Y = None, ** kwargs):

        def wrapped_layer_cost(layer, coef):
            try:
                return layer.get_weight_decay(coeff)
            except NotImplementedError:
                if coef==0.:
                    return 0.
                else:
                    raise NotImplementedError(str(type(layer))+" does not implement get_weight_decay.")


        layer_costs = [ wrapped_layer_cost(layer, coeff)
            for layer, coeff in safe_izip(model.layers, self.coeffs) ]

        assert T.scalar() != 0. # make sure theano semantics do what I want
        layer_costs = [ cost for cost in layer_costs if cost != 0.]

        if len(layer_costs) == 0:
            rval =  T.as_tensor_variable(0.)
            rval.name = '0_weight_decay'
            return rval
        else:
            total_cost = reduce(lambda x, y: x + y, layer_costs)
        total_cost.name = 'MLP_WeightDecay'

        assert total_cost.ndim == 0

        total_cost.name = 'weight_decay'

        return total_cost

    def next(self):
        next_index = self._subset_iterator.next()
        rvals = []
        for space, source, data, fn in safe_izip(self._space, self._source,
                                                 self._raw_data,
                                                 self._convert):
            rval = data[next_index]
            if isinstance(space, SequenceDataSpace):
                # Add padding
                max_sequence_length = max(len(sample) for sample
                                          in rval)
                batch = np.zeros((len(rval), max_sequence_length) +
                                 data[0].shape[1:], dtype=data[0].dtype)
                for i, sample in enumerate(rval):
                    batch[i, :len(sample)] = sample

                # Create mask
                if source in self.mask_needed:
                    mask = self._create_mask(rval)
                rval = np.swapaxes(batch, 0, 1)
                if fn:
                    rval = fn(rval)
                rvals.append(rval)
                if source in self.mask_needed:
                    rvals.append(mask)
            else:
                if fn:
                    rval = fn(rval)
                rvals.append(rval)

        # Reorder according to given data specs

        if not self._return_tuple and len(rval) == 1:
            rvals, = rvals
        return tuple(rvals)

    def get_monitoring_channels(self, X=None, Y=None):
        """
        Note: X and Y may both be None, in the case when this is
              a layer of a bigger MLP.
        """

        state = X
        rval = OrderedDict()

        for layer, scale in safe_izip(self.mlp.layers, self._params):
            state = self.scale(state, layer, scale)
            ch = layer.get_monitoring_channels()
            for key in ch:
                rval[layer.layer_name+'_'+key] = ch[key]
            state = layer.fprop(state)
            args = [state]
            if layer is self.mlp.layers[-1]:
                args.append(Y)
            ch = layer.get_monitoring_channels_from_state(*args)
            for key in ch:
                rval[layer.layer_name+'_'+key]  = ch[key]

        for i in xrange(len(self._params)):
            rval['scale_input_to_' + self.mlp.layers[i].layer_name + '_min'] = self._params[i].min()
            rval['scale_input_to_' + self.mlp.layers[i].layer_name + '_min'] = self._params[i].mean()
            rval['scale_input_to_' + self.mlp.layers[i].layer_name + '_min'] = self._params[i].max()

        return rval

    def expr(self, model, data, ** kwargs):
        """
        .. todo::

            WRITEME
        """
        self.get_data_specs(model)[0].validate(data)
        layer_costs = [layer.get_weight_decay(coeff)
                       for layer, coeff in safe_izip(model.layers,
                                                     self.coeffs)]

        assert T.scalar() != 0. # make sure theano semantics do what I want
        layer_costs = [cost for cost in layer_costs if cost != 0.]

        if len(layer_costs) == 0:
            rval = T.as_tensor_variable(0.)
            rval.name = '0_weight_decay'
            return rval
        else:
            total_cost = reduce(lambda x, y: x + y, layer_costs)
        total_cost.name = 'RNN_WeightDecay'

        assert total_cost.ndim == 0

        total_cost.name = 'weight_decay'

        return total_cost

    def add_dataset(self, dataset, mode="sequential", batch_size=None, num_batches=None, seed=None):
        """
        Determines the data used to calculate the values of each channel.

        Parameters
        ----------
        dataset : object
            A `pylearn2.datasets.Dataset` object.
        mode : str or object, optional
            Iteration mode; see the docstring of the `iterator` method
            on `pylearn2.datasets.Dataset` for details.
        batch_size : int, optional
            The size of an individual batch. Optional if `mode` is
            'sequential' and `num_batches` is specified (batch size
            will be calculated based on full dataset size).
        num_batches : int, optional
            The total number of batches. Unnecessary if `mode` is
            'sequential' and `batch_size` is specified (number of
            batches will be calculated based on full dataset size).
        """
        # The user can ommit using lists if only one dataset is set
        if not isinstance(dataset, list):
            dataset = [dataset]
        if not isinstance(mode, list):
            mode = [mode]
        if not isinstance(batch_size, list):
            batch_size = [batch_size]
        if not isinstance(num_batches, list):
            num_batches = [num_batches]
        if seed is None:
            seed = [None] * len(dataset)
        if not isinstance(seed, list):
            seed = [seed]
        if any([len(l) != len(dataset) for l in [mode, batch_size, seed]]):
            raise ValueError("make sure each dataset has its iteration " + "mode, batch size and number of batches.")
        for (d, m, b, n, sd) in safe_izip(dataset, mode, batch_size, num_batches, seed):
            try:
                it = d.iterator(mode=m, batch_size=b, num_batches=n, topo=self.topo, targets=self.require_label, rng=sd)
            except ValueError as exc:
                raise ValueError("invalid iteration parameters in " "Monitor.add_dataset: " + str(exc))
            if it.stochastic:
                # must be a seed, not a random number generator
                # if it were a random number generator, different iterators using
                # it would update its state, so we would not get the same iterator
                # each time
                # Also, must not be None, because this makes the iterator pick
                # a seed based on the clock
                if not isinstance(sd, (list, tuple, int)):
                    raise TypeError(
                        "Monitor requires a seed (not a random number generator) when using stochastic iteration modes."
                    )
            else:
                assert sd is None  # the iterator should catch this, but let's double-check

            if not d in self._datasets:
                self._datasets.append(d)
                self._iteration_mode.append(m)
                self._batch_size.append(b)
                self._num_batches.append(n)
                self._rng_seed.append(sd)

    def next(self):
        """
        Retrieves the next batch of examples.

        Returns
        -------
        next_batch : object
            An object representing a mini-batch of data, conforming
            to the space specified in the `data_specs` constructor
            argument to this iterator. Will be a tuple if more
            than one data source was specified or if the constructor
            parameter `return_tuple` was `True`.

        Raises
        ------
        StopIteration
            When there are no more batches to return.
        """
        next_index = self._subset_iterator.next()
        # TODO: handle fancy-index copies by allocating a buffer and
        # using np.take()

        rval = tuple(
            fn(data[next_index]) if fn else data[next_index]
            for data, fn in safe_izip(self._raw_data, self._convert))
        if not self._return_tuple and len(rval) == 1:
            rval, = rval
        return rval

        def getFeatures(indexes):
            """
            .. todo::
                Write me
            """
            if self._load_to_memory:
                sequences = [self.samples_sequences[i] for i in indexes]
            else:
                sequences = [self.node[i] for i in indexes]
            # Get random start point for ngram
            # Get random source word index for "ngram"
            source_i = [numpy.random.randint(self.frame_length/2 +1, len(s)-self.frame_length/2, 1)[0] 
                        for s in sequences]
            target_i = [min(abs(int(numpy.random.normal(s_i, self.frame_length/3.0))), len(s)-1)
                        for s_i, s in safe_izip(source_i, sequences)]
            preX = [s[i] for i, s in safe_izip(source_i, sequences)]
            X = []
            def make_sequence(word):
                string = self._inv_words[word]
                #if len(string) < 1:
                    #print "Word index", word, "Returns empty word"
                seq = map(lambda c: [self._char_labels.get(c, 0)], self._inv_words[word])
                #if len(seq) < 1:
                   # print "Word index", word, "Returns empty sequence", string
                seq.append([self._eow])
                return numpy.asarray(seq)

            for word in preX:
                X.append(make_sequence(word))
            X = numpy.asarray(X)
            y = [numpy.asarray([s[i]]) for i, s in safe_izip(target_i, sequences)]
            #y[y>=30000] = numpy.asarray([1])
            y = numpy.asarray(y)
            bad_is = numpy.where(y >= 30000)
            y[bad_is] =  numpy.asarray([1])
            
            # Target Words mapped to integers greater than input max are set to 
            # 1 (unknown)

            # Store the targets generated by these indices.
            self.lastY = (y, indexes)
            if self._use_words:
                self.lastPreX = preX
            return X

    def next(self):
        """
        Retrieves the next batch of examples.

        Returns
        -------
        next_batch : object
            An object representing a mini-batch of data, conforming
            to the space specified in the `data_specs` constructor
            argument to this iterator. Will be a tuple if more
            than one data source was specified or if the constructor
            parameter `return_tuple` was `True`.

        Raises
        ------
        StopIteration
            When there are no more batches to return.
        """
        next_index = self._subset_iterator.next()
        next_index = self._dataset.support[ next_index ] # !!! added line to iterate over different index set !!!

        spaces, sources = self._data_specs
        output = []                

        for data, fn, source in safe_izip(self._raw_data, self._convert, sources):
            if source=='targets':
                if fn:
                    output.append( fn(data[next_index, :]) )
                else:
                    output.append( data[next_index, :] )
            else:
                design_mat = []
                for index in next_index:                    
                    X = np.abs(data[index:index+self._dataset.tframes, :])
                    design_mat.append( X.reshape((np.prod(X.shape),)) ) 

                design_mat = np.vstack(design_mat)
                
                if self._dataset.tframes > 1:
                    # ideally we'd standardize in a preprocessing layer
                    # (so that standardization is built-in to the model rather
                    # than the dataset) but i haven't quite figured out how to do 
                    # this yet for images, due to a memory error associated with
                    # a really big diagonal scaling matrix
                    # (however, it works fine for vectors)
                    design_mat = self._dataset.standardize(design_mat)

                if fn:
                    output.append( fn(design_mat) )
                else:
                    output.append( design_mat )

        rval = tuple(output)
        if not self._return_tuple and len(rval) == 1:
            rval, = rval
        return rval

    def next(self):
        next_index = self._subset_iterator.next()
        rval = tuple(
            fn(batch) if fn else batch for batch, fn in
            safe_izip(self._dataset.get(self._source, next_index),
                      self._convert)
        )

        if not self._return_tuple and len(rval) == 1:
            rval, = rval
        return rval

def get_objs():
    n = 0.
    aves = [0. for model in models]
    m = 0
    for X, Y in train.iterator(batch_size = 5000, mode='sequential', targets=True):
        objs = [func(X, Y) for func in funcs]
        n += 1.
        aves = [ave + (obj - ave) / n for ave, obj in safe_izip(aves, objs)]
        m += X.shape[0]
    if m != 10000:
        raise AssertionError(str(m))
    return aves

    def expr(self, model, data, ** kwargs):
        """Returns a theano expression for the cost function.

        Parameters
        ----------
        model : MLP
        data : tuple
            Should be a valid occupant of
            CompositeSpace(model.get_input_space(),
            model.get_output_space())

        Returns
        -------
        total_cost : theano.gof.Variable
            coeff * sum(abs(weights))
            added up for each set of weights.
        """

        assert T.scalar() != 0.  # make sure theano semantics do what I want
        self.get_data_specs(model)[0].validate(data)
        if isinstance(self.coeffs, list):
            warnings.warn("Coefficients should be given as a dictionary "
                          "with layer names as key. The support of "
                          "coefficients as list would be deprecated "
                          "from 03/06/2015")
            layer_costs = [layer.get_l1_weight_decay(coeff)
                           for layer, coeff in safe_izip(model.layers,
                                                         self.coeffs)]
            layer_costs = [cost for cost in layer_costs if cost != 0.]

        else:
            layer_costs = []
            for layer in model.layers:
                layer_name = layer.layer_name
                if layer_name in self.coeffs:
                    cost = layer.get_l1_weight_decay(self.coeffs[layer_name])
                    if cost != 0.:
                        layer_costs.append(cost)

        if len(layer_costs) == 0:
            rval = T.as_tensor_variable(0.)
            rval.name = '0_l1_penalty'
            return rval
        else:
            total_cost = reduce(operator.add, layer_costs)
        total_cost.name = 'MLP_L1Penalty'

        assert total_cost.ndim == 0

        total_cost.name = 'l1_penalty'

        return total_cost

    def get(self, source, indexes):
        """
        Returns required examples for the required data sources, e.g. the first
        ten features and targets pairs or the last five targets

        Parameters
        ----------
        source : tuple of str
            Tuple of source names
        indexes : slice
            Examples to fetch
        """
        assert type(indexes) is slice
        # Translate indexes to stay in the [start, stop] range
        indexes = slice(indexes.start + self.start, indexes.stop + self.start,
                        indexes.step)
        # Make sure that requested sources are provided by the dataset
        self._validate_source(source)

        rval = []
        # Axes for a single example
        single_axes = [a for a in self.axes if a != 'b']
        for so in source:
            if so == 'features':
                images = self.X[indexes]
                shapes = self.s[indexes]
                space = self.data_specs[0].components[0]
                # If batch size has changed, reallocate a buffer
                if self.X_buffer is None or len(self.X_buffer) != len(images):
                    self.X_buffer = space.get_origin_batch(len(images))
                for i, (img, s) in enumerate(safe_izip(images, shapes)):
                    # Transpose image in 'b01c' format to comply with
                    # transformer interface
                    b01c = img.reshape(s).transpose(
                        [single_axes.index(a) for a in (0, 1, 'c')])
                    # Assign i'th example in the batch with the preprocessed
                    # image
                    self.X_buffer.transpose(
                        [('b', 0, 1, 'c').index(a) for a in self.axes]
                    )[i] = self.transformer(b01c)
                if self.rescale is not None:
                    self.X_buffer /= self.rescale
                rval.append(self.X_buffer)
            elif so == 'targets':
                targets = self.y[indexes]
                space = self.data_specs[0].components[1]
                # If batch size has changed, reallocate a buffer
                if self.y_buffer is None or len(self.y_buffer) != len(targets):
                    self.y_buffer = space.get_origin_batch(len(targets))
                rval.append(self.y[indexes])
        return tuple(rval)

    def expr(self, model, data, ** kwargs):
        """Returns a theano expression for the cost function.

        Parameters
        ----------
        model : MLP
        data : tuple
            Should be a valid occupant of
            CompositeSpace(model.get_input_space(),
            model.get_output_space())

        Returns
        -------
        total_cost : theano.gof.Variable
            coeff * sum(sqr(weights))
            added up for each set of weights.
        """
        self.get_data_specs(model)[0].validate(data)

        def wrapped_layer_cost(layer, coef):
            try:
                return layer.get_weight_decay(coeff)
            except NotImplementedError:
                if coef == 0.:
                    return 0.
                else:
                    reraise_as(NotImplementedError(str(type(layer)) +
                               " does not implement get_weight_decay."))

        layer_costs = [wrapped_layer_cost(layer, coeff)
                       for layer, coeff
                       in safe_izip(model.layers, self.coeffs)]

        assert T.scalar() != 0.  # make sure theano semantics do what I want
        layer_costs = [cost for cost in layer_costs if cost != 0.]

        if len(layer_costs) == 0:
            rval = T.as_tensor_variable(0.)
            rval.name = '0_weight_decay'
            return rval
        else:
            total_cost = reduce(operator.add, layer_costs)
        total_cost.name = 'MLP_WeightDecay'

        assert total_cost.ndim == 0

        total_cost.name = 'weight_decay'

        return total_cost

    def __init__(self, dataset, data_specs, subset_iterator,
                 return_tuple=False, convert=None):
        # Unpack the data specs into two tuples
        space, source = data_specs
        if not isinstance(source, tuple):
            source = (source,)

        # Remove the requested mask from the data specs before calling
        # the parent constructor
        self._original_source = source
        mask_seen, sequence_seen = False, False
        self.mask_needed = []
        retain = []
        for i, (subspace, subsource) in enumerate(safe_izip(space.components,
                                                            source)):
            if isinstance(subspace, SequenceMaskSpace):
                if not subsource.endswith('_mask') or \
                        subsource[:-5] not in source:
                    raise ValueError("SequenceDatasetIterator received "
                                     "data_specs containing a "
                                     "SequenceMaskSpace with corresponding "
                                     "source %s, but the source should end "
                                     "with `_mask` in order to match it to the"
                                     "correct SequenceDataSpace")
                mask_seen = True
                self.mask_needed.append(subsource[:-5])
            else:
                retain.append(i)
                if isinstance(subspace, SequenceDataSpace):
                    sequence_seen = True
        if mask_seen != sequence_seen and i + 1 != len(retain):
            raise ValueError("SequenceDatasetIterator was asked to iterate "
                             "over a sequence mask without data or vice versa")
        space = space.restrict(retain)
        source = tuple(source[i] for i in retain)
        super(SequenceDatasetIterator, self).__init__(
            dataset, subset_iterator, (space, source),
            return_tuple=return_tuple, convert=convert
        )
        if not isinstance(space, CompositeSpace):
            space = (space,)
        else:
            space = space.components
        assert len(space) == len(source)
        self._original_space = space

    def fprop(self, state_below, apply_dropout = False):

        if apply_dropout:
            warnings.warn("dropout should be implemented with fixed_var_descr to make sure it works with BGD, this is just a hack to get it working with SGD")
            theano_rng = MRG_RandomStreams(self.rng.randint(2**15))
            state_below = self.apply_dropout(state=state_below, include_prob=self.dropout_input_include_prob, theano_rng=theano_rng)

        rval = self.layers[0].fprop(state_below)

        if apply_dropout:
            dropout = self.dropout_include_probs[0]
            rval = self.apply_dropout(state=rval, include_prob=dropout, theano_rng=theano_rng)

        for layer, dropout in safe_izip(self.layers[1:], self.dropout_include_probs[1:]):
            rval = layer.fprop(rval)
            if apply_dropout:
                rval = self.apply_dropout(state=rval, include_prob=dropout, theano_rng=theano_rng)

        return rval

    def __call__(self, model, X, Y = None, ** kwargs):

        layer_costs = [ layer.get_weight_decay(coeff)
            for layer, coeff in safe_izip(model.hidden_layers, self.coeffs) ]

        assert T.scalar() != 0. # make sure theano semantics do what I want
        layer_costs = [ cost for cost in layer_costs if cost != 0.]

        if len(layer_costs) == 0:
            rval =  T.as_tensor_variable(0.)
            rval.name = '0_weight_decay'
            return rval
        else:
            total_cost = reduce(lambda x, y: x + y, layer_costs)
        total_cost.name = 'DBM_WeightDecay'

        assert total_cost.ndim == 0

        total_cost.name = 'weight_decay'

        return total_cost