/// <summary>
        ///     Analyze the data. This counts the records and prepares the data to be
        ///     processed.
        /// </summary>
        /// <param name="theAnalyst">The analyst to use.</param>
        /// <param name="inputFile">The input file.</param>
        /// <param name="headers">True if headers are present.</param>
        /// <param name="format">The format the file is in.</param>
        public void Analyze(EncogAnalyst theAnalyst,
                            FileInfo inputFile, bool headers, CSVFormat format)
        {
            InputFilename = inputFile;
            ExpectInputHeaders = headers;
            Format = format;
            _analyst = theAnalyst;

            Analyzed = true;

            PerformBasicCounts();

            _inputCount = _analyst.DetermineInputCount();
            _outputCount = _analyst.DetermineOutputCount();
            _idealCount = InputHeadings.Length - _inputCount;

            if ((InputHeadings.Length != _inputCount)
                && (InputHeadings.Length != (_inputCount + _outputCount)))
            {
                throw new AnalystError("Invalid number of columns("
                                       + InputHeadings.Length + "), must match input("
                                       + _inputCount + ") count or input+output("
                                       + (_inputCount + _outputCount) + ") count.");
            }
        }

        /// <summary>
        ///     Construct the object.
        /// </summary>
        /// <param name="theAnalyst">The analyst.</param>
        /// <param name="theBackwardWindowSize">The backward window size.</param>
        /// <param name="theForwardWindowSize">The forward window size.</param>
        public AnalystProcess(EncogAnalyst theAnalyst, int theBackwardWindowSize, int theForwardWindowSize)
        {
            analyst = theAnalyst;

            backwardWindowSize = theBackwardWindowSize;
            forwardWindowSize = theForwardWindowSize;
            StandardExtensions.CreateAll(programContext);
        }

 /// <inheritdoc/>
 public double[] HandleMissing(EncogAnalyst analyst, AnalystField stat)
 {
     var result = new double[stat.ColumnsNeeded];
     double n = stat.NormalizedHigh - (stat.NormalizedHigh - stat.NormalizedLow/2);
     for (int i = 0; i < result.Length; i++)
     {
         result[i] = n;
     }
     return result;
 }

 /// <inheritdoc/>
 public double[] HandleMissing(EncogAnalyst analyst, AnalystField stat)
 {
     // mode?
     if (stat.Classify)
     {
         var m = stat.DetermineMode(analyst);
         return stat.Encode(m);
     }
     // mean
     var df = analyst.Script.FindDataField(stat.Name);
     var result = new double[1];
     result[0] = df.Mean;
     return result;
 }

        /// <summary>
        /// Analyze the data. This counts the records and prepares the data to be
        /// processed.
        /// </summary>
        ///
        /// <param name="theAnalyst">The analyst to use.</param>
        /// <param name="inputFile">The input file to analyze.</param>
        /// <param name="headers">True, if the input file has headers.</param>
        /// <param name="format">The format of the input file.</param>
        public void Analyze(EncogAnalyst theAnalyst,
                            FileInfo inputFile, bool headers, CSVFormat format)
        {
            InputFilename = inputFile;
            ExpectInputHeaders = headers;
            InputFormat = format;

            Analyzed = true;
            _analyst = theAnalyst;

            if (OutputFormat == null)
            {
                OutputFormat = InputFormat;
            }

            _data = new BasicMLDataSet();
            ResetStatus();
            int recordCount = 0;

            int outputLength = _analyst.DetermineTotalColumns();
            var csv = new ReadCSV(InputFilename.ToString(),
                                  ExpectInputHeaders, InputFormat);
            ReadHeaders(csv);

            _analystHeaders = new CSVHeaders(InputHeadings);

            while (csv.Next() && !ShouldStop())
            {
                UpdateStatus(true);

                var row = new LoadedRow(csv, 1);

                double[] inputArray = AnalystNormalizeCSV.ExtractFields(
                    _analyst, _analystHeaders, csv, outputLength, true);
                var input = new ClusterRow(inputArray, row);
                _data.Add(input);

                recordCount++;
            }
            RecordCount = recordCount;
            Count = csv.ColumnCount;

            ReadHeaders(csv);
            csv.Close();
            ReportDone(true);
        }

        /// <summary>
        /// Analyze the data. This counts the records and prepares the data to be
        /// processed.
        /// </summary>
        ///
        /// <param name="theAnalyst">The analyst to use.</param>
        /// <param name="inputFile">The input file.</param>
        /// <param name="headers">True if headers are present.</param>
        /// <param name="format">The format.</param>
        public void Analyze(EncogAnalyst theAnalyst,
                            FileInfo inputFile, bool headers, CSVFormat format)
        {
            InputFilename = inputFile;
            ExpectInputHeaders = headers;
            Format = format;

            Analyzed = true;
            _analyst = theAnalyst;

            PerformBasicCounts();
            _fileColumns = InputHeadings.Length;
            _outputColumns = _analyst.DetermineOutputFieldCount();

            _analystHeaders = new CSVHeaders(InputHeadings);
            _series = new TimeSeriesUtil(_analyst, false,
                                        _analystHeaders.Headers);
        }

        /// <summary>
        ///     Construct the time-series utility.
        /// </summary>
        /// <param name="theAnalyst">The analyst to use.</param>
        /// <param name="includeOutput">Should output fields be included.</param>
        /// <param name="headings">The column headings.</param>
        public TimeSeriesUtil(EncogAnalyst theAnalyst, bool includeOutput,
                              IEnumerable<string> headings)
        {
            _buffer = new List<double[]>();
            _headingMap = new Dictionary<String, Int32>();
            _analyst = theAnalyst;
            _lagDepth = _analyst.LagDepth;
            _leadDepth = _analyst.LeadDepth;
            _totalDepth = _lagDepth + _leadDepth + 1;
            _inputSize = includeOutput ? _analyst.DetermineTotalColumns() : _analyst.DetermineTotalInputFieldCount();
            _outputSize = _analyst.DetermineInputCount()
                          + _analyst.DetermineOutputCount();

            int headingIndex = 0;

            foreach (String column  in  headings)
            {
                _headingMap[column.ToUpper()] = headingIndex++;
            }
        }

 /// <summary>
 ///     Construct this generate command.
 /// </summary>
 /// <param name="analyst">The analyst to use.</param>
 public CmdCode(EncogAnalyst analyst)
     : base(analyst)
 {
 }

 /// <summary>
 ///     Construct the train command.
 /// </summary>
 /// <param name="analyst">The analyst to use.</param>
 public CmdTrain(EncogAnalyst analyst)
     : base(analyst)
 {
 }

 /// <summary>
 ///     Construct the analyst wizard.
 /// </summary>
 /// <param name="theAnalyst">The analyst to use.</param>
 public AnalystWizard(EncogAnalyst theAnalyst)
 {
     _directClassification = false;
     _taskSegregate = true;
     _taskRandomize = true;
     _taskNormalize = true;
     _taskBalance = false;
     _taskCluster = true;
     _range = NormalizeRange.NegOne2One;
     _analyst = theAnalyst;
     _script = _analyst.Script;
     _methodType = WizardMethodType.FeedForward;
     TargetFieldName = "";
     _goal = AnalystGoal.Classification;
     _leadWindowSize = 0;
     _lagWindowSize = 0;
     _includeTargetField = false;
     _missing = new DiscardMissing();
     MaxError = DefaultTrainError;
     NaiveBayes = false;
 }

        /// <summary>
        /// Determine the mode, this is the class item that has the most instances.
        /// </summary>
        /// <param name="analyst">The analyst to use.</param>
        /// <returns>The mode.</returns>
        public int DetermineMode(EncogAnalyst analyst)
        {
            if (!Classify)
            {
                throw new AnalystError("Can only calculate the mode for a class.");
            }

            DataField df = analyst.Script.FindDataField(Name);
            AnalystClassItem m = null;
            int result = 0;
            int idx = 0;
            foreach (AnalystClassItem item in df.ClassMembers)
            {
                if (m == null || m.Count < item.Count)
                {
                    m = item;
                    result = idx;
                }
                idx++;
            }

            return result;
        }

 /// <summary>
 /// Construct this command.
 /// </summary>
 ///
 /// <param name="theAnalyst">The analyst that this command belongs to.</param>
 protected Cmd(EncogAnalyst theAnalyst)
 {
     _analyst = theAnalyst;
     _script = _analyst.Script;
     _properties = _script.Properties;
 }

 /// <summary>
 ///     Construct the create command.
 /// </summary>
 /// <param name="theAnalyst">The analyst to use.</param>
 public CmdCreate(EncogAnalyst theAnalyst) : base(theAnalyst)
 {
 }

 /// <summary>
 /// Construct the evaluate command.
 /// </summary>
 ///
 /// <param name="theAnalyst">The analyst to use.</param>
 public CmdEvaluate(EncogAnalyst theAnalyst)
     : base(theAnalyst)
 {
 }

 /// <summary>
 /// Construct the segregate command.
 /// </summary>
 ///
 /// <param name="analyst">The analyst to use.</param>
 public CmdSegregate(EncogAnalyst analyst)
     : base(analyst)
 {
 }

        /// <summary>
        /// Extract fields from a file into a numeric array for machine learning.
        /// </summary>
        ///
        /// <param name="analyst">The analyst to use.</param>
        /// <param name="headers">The headers for the input data.</param>
        /// <param name="csv">The CSV that holds the input data.</param>
        /// <param name="outputLength">The length of the returned array.</param>
        /// <param name="skipOutput">True if the output should be skipped.</param>
        /// <returns>The encoded data.</returns>
        public static double[] ExtractFields(EncogAnalyst analyst,
                                             CSVHeaders headers, ReadCSV csv,
                                             int outputLength, bool skipOutput)
        {
            var output = new double[outputLength];
            int outputIndex = 0;

            foreach (AnalystField stat in analyst.Script.Normalize.NormalizedFields)
            {
                if (stat.Action == NormalizationAction.Ignore)
                {
                    continue;
                }

                if (stat.Output && skipOutput)
                {
                    continue;
                }

                int index = headers.Find(stat.Name);
                String str = csv.Get(index);

                // is this an unknown value?
                if (str.Equals("?") || str.Length == 0)
                {
                    IHandleMissingValues handler = analyst.Script.Normalize.MissingValues;
                    double[] d = handler.HandleMissing(analyst, stat);

                    // should we skip the entire row
                    if (d == null)
                    {
                        return null;
                    }

                    // copy the returned values in place of the missing values
                    for (int i = 0; i < d.Length; i++)
                    {
                        output[outputIndex++] = d[i];
                    }
                }
                else
                {
                    // known value

                    if (stat.Action == NormalizationAction.Normalize)
                    {
                        double d = csv.Format.Parse(str.Trim());
                        d = stat.Normalize(d);
                        output[outputIndex++] = d;
                    }
                    else
                    {
                        double[] d = stat.Encode(str.Trim());

                        foreach (double element in d)
                        {
                            output[outputIndex++] = element;
                        }
                    }
                }
            }

            return output;
        }

        /// <summary>
        /// Analyze the file.
        /// </summary>
        ///
        /// <param name="inputFilename">The input file.</param>
        /// <param name="expectInputHeaders">True, if input headers are present.</param>
        /// <param name="inputFormat">The format.</param>
        /// <param name="theAnalyst">The analyst to use.</param>
        public void Analyze(FileInfo inputFilename,
                            bool expectInputHeaders, CSVFormat inputFormat,
                            EncogAnalyst theAnalyst)
        {
            InputFilename = inputFilename;
            InputFormat = inputFormat;
            ExpectInputHeaders = expectInputHeaders;
            _analyst = theAnalyst;
            Analyzed = true;

            _analystHeaders = new CSVHeaders(inputFilename, expectInputHeaders,
                                            inputFormat);


            foreach (AnalystField field  in  _analyst.Script.Normalize.NormalizedFields)
            {
                field.Init();
            }

            _series = new TimeSeriesUtil(_analyst,
                                        _analystHeaders.Headers);
        }

 /// <summary>
 /// Construct the report.
 /// </summary>
 ///
 /// <param name="theAnalyst">The analyst to use.</param>
 public AnalystReport(EncogAnalyst theAnalyst)
 {
     _analyst = theAnalyst;
 }

 public double[] HandleMissing(EncogAnalyst analyst, AnalystField stat)
 {
     return null;
 }

 /// <summary>
 ///     Construct the set command with the analyst.
 /// </summary>
 /// <param name="analyst">The analyst to use.</param>
 public CmdSet(EncogAnalyst analyst)
     : base(analyst)
 {
 }