/// <summary>
        /// Construct a neural genetic algorithm.
        /// </summary>
        ///
        /// <param name="network">The network to base this on.</param>
        /// <param name="randomizer">The randomizer used to create this initial population.</param>
        /// <param name="calculateScore">The score calculation object.</param>
        /// <param name="populationSize">The population size.</param>
        /// <param name="mutationPercent">The percent of offspring to mutate.</param>
        /// <param name="percentToMate">The percent of the population allowed to mate.</param>
        public NeuralGeneticAlgorithm(BasicNetwork network,
            IRandomizer randomizer, ICalculateScore calculateScore,
            int populationSize, double mutationPercent,
            double percentToMate)
            : base(TrainingImplementationType.Iterative)
        {
            Genetic = new NeuralGeneticAlgorithmHelper
                           {
                               CalculateScore = new GeneticScoreAdapter(calculateScore)
                           };
            IPopulation population = new BasicPopulation(populationSize);
            Genetic.MutationPercent = mutationPercent;
            Genetic.MatingPopulation = percentToMate*2;
            Genetic.PercentToMate = percentToMate;
            Genetic.Crossover = new Splice(network.Structure.CalculateSize()/3);
            Genetic.Mutate = new MutatePerturb(4.0d);
            Genetic.Population = population;
            for (int i = 0; i < population.PopulationSize; i++)
            {
                var chromosomeNetwork = (BasicNetwork) (network
                                                           .Clone());
                randomizer.Randomize(chromosomeNetwork);

                var genome = new NeuralGenome(chromosomeNetwork) {GA = Genetic};
                Genetic.PerformCalculateScore(genome);
                Genetic.Population.Add(genome);
            }
            population.Sort();
        }

 /// <summary>
 /// Construct a Manhattan propagation training object.
 /// </summary>
 ///
 /// <param name="network">The network to train.</param>
 /// <param name="training">The training data to use.</param>
 /// <param name="learnRate">The learning rate.</param>
 public ManhattanPropagation(BasicNetwork network,
     IMLDataSet training, double learnRate)
     : base(network, training)
 {
     _learningRate = learnRate;
     _zeroTolerance = RPROPConst.DefaultZeroTolerance;
 }

        /// <summary>
        /// Format the network as a human readable string that lists the 
        /// hidden layers.
        /// </summary>
        /// <param name="network">The network to format.</param>
        /// <returns>A human readable string.</returns>
        public static String NetworkToString(BasicNetwork network)
        {
            StringBuilder result = new StringBuilder();
            int num = 1;

            ILayer layer = network.GetLayer(BasicNetwork.TAG_INPUT);

            // display only hidden layers
            while (layer.Next.Count > 0)
            {
                layer = layer.Next[0].ToLayer;

                if (result.Length > 0)
                {
                    result.Append(",");
                }
                result.Append("H");
                result.Append(num++);
                result.Append("=");
                result.Append(layer.NeuronCount);
            }

            return result.ToString();

        }

 /// <summary>
 /// Construct a Hopfield training class.
 /// </summary>
 /// <param name="trainingSet">The training set to use.</param>
 /// <param name="network">The network to train.</param>
 public TrainHopfield(INeuralDataSet trainingSet,
          BasicNetwork network)
 {
     this.network = network;
     this.Training = trainingSet;
     this.Error = 0;
 }

 /// <inheritdoc />
 public override void Randomize(BasicNetwork network)
 {
     for (var i = 0; i < network.Layers.Count - 1; i++)
     {
         RandomizeLayer(network, i);
     }
 }

 /// <summary>
 /// Construct the depth calculation object.
 /// </summary>
 /// <param name="network">The network that we are calculating for.</param>
 public CalculateDepth(BasicNetwork network)
 {
     this.network = network;
     this.outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT);
     if( this.outputLayer!=null )
         Calculate(0, this.outputLayer);
 }

 /// <summary>
 /// Construct an RPROP job. For more information on RPROP see the
 /// ResilientPropagation class. 
 /// </summary>
 /// <param name="network">The network to train.</param>
 /// <param name="training">The training data to use.</param>
 /// <param name="loadToMemory">True if binary training data should be loaded to memory.</param>
 /// <param name="localRatio">The local ratio, used if this job is performed by an OpenCL Device.</param>
 /// <param name="globalRatio">The global ratio, used if this job is performed by an OpenCL Device.</param>
 /// <param name="segmentationRatio">The segmentation ratio, used if this job is performed by an OpenCL Device.</param>
 /// <param name="iterationsPer">How many iterations to process per cycle.</param>
 public RPROPJob(BasicNetwork network, INeuralDataSet training,
         bool loadToMemory, double localRatio, int globalRatio, double segmentationRatio, int iterationsPer) :
     this(network, training,
          loadToMemory, RPROPConst.DEFAULT_INITIAL_UPDATE,
          RPROPConst.DEFAULT_MAX_STEP, localRatio, globalRatio, segmentationRatio, iterationsPer)
 {
 }

 /// <summary>
 /// Construct a cross trainer. 
 /// </summary>
 /// <param name="network">The network.</param>
 /// <param name="training">The training data.</param>
 public CrossTraining(BasicNetwork network,
          FoldedDataSet training)
 {
     this.network = network;
     Training = training;
     this.folded = training;
 }

 /// <inheritdoc />
 public override void Randomize(BasicNetwork network)
 {
     for (var i = 0; i < network.Weights.Length; i++)
     {
         network.Weights[i] = Rnd.NextDouble(_low, _high);
     }
 }

        /// <summary>
        /// Construct the LMA object.
        /// </summary>
        /// <param name="network">The network to train. Must have a single output neuron.</param>
        /// <param name="training">The training data to use. Must be indexable.</param>
        public SVDTraining(BasicNetwork network, INeuralDataSet training)
        {
            ILayer outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT);

            if (outputLayer == null)
            {
                throw new TrainingError("SVD requires an output layer.");
            }

            if (outputLayer.NeuronCount != 1)
            {
                throw new TrainingError("SVD requires an output layer with a single neuron.");
            }

            if (network.GetLayer(RadialBasisPattern.RBF_LAYER) == null)
                throw new TrainingError("SVD is only tested to work on radial basis function networks.");

            rbfLayer = (RadialBasisFunctionLayer)network.GetLayer(RadialBasisPattern.RBF_LAYER);

            this.Training = training;
            this.network = network;
            this.trainingLength = (int)this.Training.InputSize;

            BasicNeuralData input = new BasicNeuralData(this.Training.InputSize);
            BasicNeuralData ideal = new BasicNeuralData(this.Training.IdealSize);
            this.pair = new BasicNeuralDataPair(input, ideal);
        }

        /// <summary>
        /// Determine if the two neural networks are equal.
        /// </summary>
        ///
        /// <param name="network1">The first network.</param>
        /// <param name="network2">The second network.</param>
        /// <param name="precision">How many decimal places to check.</param>
        /// <returns>True if the two networks are equal.</returns>
        public static bool Equals(BasicNetwork network1,
                                  BasicNetwork network2, int precision)
        {
            double[] array1 = NetworkToArray(network1);
            double[] array2 = NetworkToArray(network2);

            if (array1.Length != array2.Length)
            {
                return false;
            }

            double test = Math.Pow(10.0d, precision);
            if (Double.IsInfinity(test) || (test > Int64.MaxValue))
            {
                throw new NeuralNetworkError("Precision of " + precision
                                             + " decimal places is not supported.");
            }

            for (int i = 0; i < array1.Length; i++)
            {
                var l1 = (long) (array1[i]*test);
                var l2 = (long) (array2[i]*test);
                if (l1 != l2)
                {
                    return false;
                }
            }

            return true;
        }

 /// <summary>
 /// Construct a QPROP trainer for flat networks.
 /// </summary>
 /// <param name="network">The network to train.</param>
 /// <param name="training">The training data.</param>
 /// <param name="learnRate">The learning rate.  2 is a good suggestion as 
 ///            a learning rate to start with.  If it fails to converge, 
 ///            then drop it.  Just like backprop, except QPROP can 
 ///            take higher learning rates.</param>
 public QuickPropagation(BasicNetwork network,
                         IMLDataSet training, double learnRate)
     : base(network, training)
 {
     ValidateNetwork.ValidateMethodToData(network, training);
     LearningRate = learnRate;
 }

        /// <summary>
        /// Determine the network size.
        /// </summary>
        /// <param name="network">The network to check.</param>
        /// <returns>The size of the network.</returns>
        public static int NetworkSize(BasicNetwork network)
        {

            // see if there is already an up to date flat network
            if (network.Structure.Flat != null
                && (network.Structure.FlatUpdate == FlatUpdateNeeded.None
                || network.Structure.FlatUpdate == FlatUpdateNeeded.Unflatten))
            {
                return network.Structure.Flat.Weights.Length;
            }

            int index = 0;

            // loop over all of the layers, take the output layer first
            foreach (ILayer layer in network.Structure.Layers)
            {

                // see if the previous layer, which is the next layer that the loop will hit,
                // is either a connection to a BasicLayer or a ContextLayer.
                ISynapse synapse = network.Structure
                        .FindPreviousSynapseByLayerType(layer, typeof(BasicLayer));
                ISynapse contextSynapse = network.Structure.FindPreviousSynapseByLayerType(
                        layer, typeof(ContextLayer));

                // get a list of of the previous synapses to this layer
                IList<ISynapse> list = network.Structure.GetPreviousSynapses(layer);

                // If there is not a BasicLayer or contextLayer as the next layer, then
                // just take the first synapse of any type.
                if (synapse == null && contextSynapse == null && list.Count > 0)
                {
                    synapse = list[0];
                }

                // is there any data to record for this synapse?
                if (synapse != null && synapse.WeightMatrix != null)
                {
                    // process each weight matrix
                    for (int x = 0; x < synapse.ToNeuronCount; x++)
                    {

                        index += synapse.FromNeuronCount;


                        if (synapse.ToLayer.HasBias)
                        {
                            index++;
                        }

                        if (contextSynapse != null)
                        {
                            index += contextSynapse.FromNeuronCount;
                        }
                    }
                }
            }

            return index;
        }

 /// <summary>
 /// Construct the instar training object.
 /// </summary>
 /// <param name="network">The network to be trained.</param>
 /// <param name="training">The training data.</param>
 /// <param name="learningRate">The learning rate.</param>
 public TrainInstar(BasicNetwork network, INeuralDataSet training,
         double learningRate)
 {
     this.network = network;
     this.training = training;
     this.learningRate = learningRate;
     this.parts = new FindCPN(network);
 }

 /// <summary>
 /// Construct a training class.
 /// </summary>
 /// <param name="network">The network to train.</param>
 /// <param name="training">The training data.</param>
 public ScaledConjugateGradient(BasicNetwork network,
         INeuralDataSet training)
     : base(network, training)
 {
     TrainFlatNetworkSCG rpropFlat = new TrainFlatNetworkSCG(
             network.Structure.Flat,
             this.Training);
     this.FlatTraining = rpropFlat;
 }

 /// <summary>
 /// Construct a QPROP trainer for flat networks.
 /// </summary>
 /// <param name="network">The network to train.</param>
 /// <param name="training">The training data.</param>
 /// <param name="learnRate">The learning rate.  2 is a good suggestion as 
 ///            a learning rate to start with.  If it fails to converge, 
 ///            then drop it.  Just like backprop, except QPROP can 
 ///            take higher learning rates.</param>
 public QuickPropagation(BasicNetwork network,
     IMLDataSet training, double learnRate)
     : base(network, training)
 {
     ValidateNetwork.ValidateMethodToData(network, training);
     LearningRate = learnRate;
     LastDelta = new double[Network.Flat.Weights.Length];
     OutputEpsilon = 1.0;
 }

        /// <summary>
        /// Construct a job definition for RPROP. For more information on backprop,
        /// see the Backpropagation class. 
        /// </summary>
        /// <param name="network">The network to use.</param>
        /// <param name="training">The training data to use.</param>
        /// <param name="loadToMemory">Should binary data be loaded to memory?</param>
        /// <param name="learningRate">THe learning rate to use.</param>
        /// <param name="momentum">The momentum to use.</param>
        public BPROPJob(BasicNetwork network, INeuralDataSet training,
                 bool loadToMemory, double learningRate,
                 double momentum)
            : base(network, training, loadToMemory)
        {

            this.LearningRate = learningRate;
            this.Momentum = momentum;
        }

        /// <summary>
        /// Use an array to populate the memory of the neural network.
        /// </summary>
        /// <param name="array">An array of doubles.</param>
        /// <param name="network">The network to encode.</param>
        public static void ArrayToNetwork(double[] array,
                 BasicNetwork network)
        {
            int index = 0;

            foreach (ILayer layer in network.Structure.Layers)
            {
                index = NetworkCODEC.ProcessLayer(network, layer, array, index);
            }

            network.Structure.FlatUpdate = FlatUpdateNeeded.Flatten;
        }

        /// <summary>
        /// Construct an ADALINE trainer.
        /// </summary>
        ///
        /// <param name="network">The network to train.</param>
        /// <param name="training">The training data.</param>
        /// <param name="learningRate">The learning rate.</param>
        public TrainAdaline(BasicNetwork network, IMLDataSet training,
                            double learningRate) : base(TrainingImplementationType.Iterative)
        {
            if (network.LayerCount > 2)
            {
                throw new NeuralNetworkError(
                    "An ADALINE network only has two layers.");
            }
            _network = network;

            _training = training;
            _learningRate = learningRate;
        }

 /// <summary>
 ///     Construct the backpropagation trainer.
 /// </summary>
 /// <param name="theNetwork">The network to train.</param>
 /// <param name="theTraining">The training data to use.</param>
 /// <param name="theLearningRate">The learning rate.  Can be changed as training runs.</param>
 /// <param name="theMomentum">The momentum.  Can be changed as training runs.</param>
 public BackPropagation(BasicNetwork theNetwork, IList<BasicData> theTraining, double theLearningRate,
     double theMomentum)
 {
     BatchSize = 500;
     Stochastic = new MersenneTwisterGenerateRandom();
     NesterovUpdate = true;
     _network = theNetwork;
     _training = theTraining;
     LearningRate = theLearningRate;
     Momentum = theMomentum;
     _gradients = new GradientCalc(_network, new CrossEntropyErrorFunction(), this);
     _lastDelta = new double[theNetwork.Weights.Length];
 }