public void Build(MetricDB db, int k, Index ref_index)
        {
            this.DB = db;
            this.K = k;
            this.R = ref_index;
            int sigma = this.R.DB.Count;
            this.INVINDEX = new List<List<int>> (sigma);
            for (int i = 0; i < sigma; ++i) {
                this.INVINDEX.Add(new List<int>());
            }
            var A = new int[this.DB.Count][];
            int count = 0;
            var compute_one = new Action<int>(delegate(int objID) {
                var u = this.GetKnr(this.DB[objID], this.K);
                A[objID] = u;
                ++count;
                if (count % 1000 == 0) {
                    Console.WriteLine ("==== {0}/{1} db: {2}, k: {3}", count, this.DB.Count, this.DB.Name, k);
                }
            });
            ParallelOptions ops = new ParallelOptions();
            ops.MaxDegreeOfParallelism = -1;
            Parallel.ForEach(new ListGen<int>((int i) => i, this.DB.Count), ops, compute_one);

            for (int objID = 0; objID < this.DB.Count; ++objID) {
                var u = A[objID];
                for (int i = 0; i < this.K; ++i) {
                    this.INVINDEX[u[i]].Add (objID);
                }
            }
        }

        public void ImportAllStreams()
        {
            var ids = new List<int>();
            using (var con = new System.Data.SqlClient.SqlConnection(_cs))
            {
                con.Open();
                ids = GetStreamIds(con).OrderBy(d => d).ToList();
            }
            var po = new ParallelOptions { MaxDegreeOfParallelism = 4 };
            //Parallel.ForEach(ids, po, id =>
            //{
            //    var sw = Stopwatch.StartNew();
            //    TryImportStream(id);
            //    sw.Stop();
            //    ImportedStreams++;

            //    Console.WriteLine(" Done in {0}", sw.Elapsed);
            //});
            foreach (var id in ids)
            {
                var sw = Stopwatch.StartNew();

                TryImportStream(id);
                sw.Stop();
                ImportedStreams++;

                Console.WriteLine(" Done in {0}", sw.Elapsed);
            }
        }

        public List<BuildInfoDto> GetBuildInfoDtos()
        {
            var buildInfoDtos = new ConcurrentBag<BuildInfoDto>();
            try
            {
                
                var tfsServer = _helperClass.GetTfsServer();
                // Get the catalog of team project collections
                var teamProjectCollectionNodes = tfsServer.CatalogNode.QueryChildren(
                    new[] { CatalogResourceTypes.ProjectCollection }, false, CatalogQueryOptions.None);
                var parallelOptions = new ParallelOptions {MaxDegreeOfParallelism = 1};
                Parallel.ForEach(teamProjectCollectionNodes, parallelOptions, teamProjectCollectionNode =>
                {
                    
                        var task = GetBuildInfoDtosPerTeamProject(teamProjectCollectionNode, tfsServer, DateTime.MinValue);
                        task.ConfigureAwait(false);
                        task.Wait();    
                        var buildInfos = task.Result;

                        foreach (var buildInfoDto in buildInfos)
                        {
                            buildInfoDtos.Add(buildInfoDto);
                        }
                        
                    
                });
            }
            catch (Exception e)
            {
                LogService.WriteError(e);
                throw;
            }

            return buildInfoDtos.ToList();
        }

        public List<BuildInfoDto> GetBuildInfoDtosPolling(String filterDate)
        {
            var buildInfoDtos = new List<BuildInfoDto>();
            try
            {
                var sinceDateTime = DateTime.Now.Subtract(new TimeSpan(int.Parse(filterDate), 0, 0));


                var tfsServer = _helperClass.GetTfsServer();

                // Get the catalog of team project collections
                var teamProjectCollectionNodes = tfsServer.CatalogNode.QueryChildren(
                    new[] { CatalogResourceTypes.ProjectCollection }, false, CatalogQueryOptions.None);

                var parallelOptions = new ParallelOptions() {MaxDegreeOfParallelism = 1};
                Parallel.ForEach(teamProjectCollectionNodes, parallelOptions, teamProjectCollectionNode =>
                {
                    var taskBuidInfos = GetBuildInfoDtosPerTeamProject(teamProjectCollectionNode, tfsServer, sinceDateTime);
                    taskBuidInfos.ConfigureAwait(false);
                    taskBuidInfos.Wait();

                    lock (buildInfoDtos)
                    {
                        buildInfoDtos.AddRange(taskBuidInfos.Result);
                    }
                });
            }
            catch (Exception e)
            {
                LogService.WriteError(e);
                throw;
            }

            return buildInfoDtos;
        }

        /// <summary>
        ///     Initialize the experiment with configuration file parameters.
        /// </summary>
        /// <param name="name">The name of the experiment</param>
        /// <param name="xmlConfig">The parent XML configuration element</param>
        public virtual void Initialize(string name, XmlElement xmlConfig)
        {
            // Set all properties
            Name = name;
            DefaultPopulationSize = XmlUtils.GetValueAsInt(xmlConfig, "PopulationSize");
            Description = XmlUtils.GetValueAsString(xmlConfig, "Description");

            // Set all internal class variables
            _activationScheme = ExperimentUtils.CreateActivationScheme(xmlConfig, "Activation");
            ComplexityRegulationStrategy = XmlUtils.TryGetValueAsString(xmlConfig, "ComplexityRegulationStrategy");
            Complexitythreshold = XmlUtils.TryGetValueAsInt(xmlConfig, "ComplexityThreshold");
            ParallelOptions = ExperimentUtils.ReadParallelOptions(xmlConfig);

            // Set evolution/genome parameters
            NeatEvolutionAlgorithmParameters = new NeatEvolutionAlgorithmParameters
            {
                SpecieCount = XmlUtils.GetValueAsInt(xmlConfig, "SpecieCount"),
                InterspeciesMatingProportion = XmlUtils.GetValueAsDouble(xmlConfig,
                    "InterspeciesMatingProbability"),
                MinTimeAlive = XmlUtils.GetValueAsInt(xmlConfig, "MinTimeAlive")
            };
            NeatGenomeParameters = ExperimentUtils.ReadNeatGenomeParameters(xmlConfig);

            // Set experiment-specific parameters
            MaxTimesteps = XmlUtils.TryGetValueAsInt(xmlConfig, "MaxTimesteps");
            MinSuccessDistance = XmlUtils.TryGetValueAsInt(xmlConfig, "MinSuccessDistance");
            MaxDistanceToTarget = XmlUtils.TryGetValueAsInt(xmlConfig, "MaxDistanceToTarget");
            MazeVariant =
                MazeVariantUtl.convertStringToMazeVariant(XmlUtils.TryGetValueAsString(xmlConfig, "MazeVariant"));         
        }

        static void Main(string[] args)
        {
            TaskScheduler.UnobservedTaskException += OnUnobservedTaskException;

            Console.WriteLine("Press enter to start");
            Console.ReadLine();

            // Batch the connections, once they're all active though it's fine
            int connectionBatchSize = 10;
            int connectionSleepInterval = 1000;
            int maxClients = 3000;

            var options = new ParallelOptions
            {
                MaxDegreeOfParallelism = connectionBatchSize
            };

            for (int x = 0; x < maxClients; x += connectionBatchSize)
            {
                Parallel.For(0, connectionBatchSize, options, i =>
                    {
                        StartClient();
                    });

                System.Threading.Thread.Sleep(connectionSleepInterval);
            }

            Console.Read();
        }

        public void ValidateSparseMatrixAsDenseAnsi()
        {
            SparseMatrix<string, string, double> sparseMatrixObj = GetSparseMatrix();

            ParallelOptions parallelOptObj = new ParallelOptions();

            DenseAnsi denseAnsiObj =
                sparseMatrixObj.AsDenseAnsi<double>(parallelOptObj);

            // Validate all properties of Ansi and SparseMatrix
            Assert.AreEqual(sparseMatrixObj.ColCount, denseAnsiObj.ColCount);
            Assert.AreEqual(sparseMatrixObj.ColKeys.Count, denseAnsiObj.ColKeys.Count);
            Assert.AreEqual(sparseMatrixObj.IndexOfColKey.Count, denseAnsiObj.IndexOfColKey.Count);
            Assert.AreEqual(sparseMatrixObj.IndexOfRowKey.Count, denseAnsiObj.IndexOfRowKey.Count);
            Assert.AreEqual("?", denseAnsiObj.MissingValue.ToString((IFormatProvider)null));
            Assert.AreEqual(sparseMatrixObj.RowCount, denseAnsiObj.RowCount);
            Assert.AreEqual(sparseMatrixObj.RowKeys.Count, denseAnsiObj.RowKeys.Count);
            Assert.AreEqual(utilityObj.xmlUtil.GetTextValue(
                Constants.SimpleMatrixNodeName,
                Constants.DenseAnsiStringNode),
                denseAnsiObj.ToString2D().Replace("\r", "").Replace("\n", "").Replace("\t", ""));

            ApplicationLog.WriteLine(
                "SparseMatrix BVT : Validation of AsDenseAnsi() method successful");
        }

        public override async Task ExecuteAsync()
        {
            // Create 100 Data objects
            var data = Enumerable.Range(1, 100).Select(i => new Data { Number = i });
            
            // Option 1a: use Upsert with an IDictionary<string, object> of documents in upsert
            var bulkData = data.ToDictionary(d => "dotnetDevguideExample-" + d.Number);
            // Note: There is no UpsertAsync overload that takes multiple values (yet)
            // which is why this example wraps the synchronized method in a new Task.
            // If your code is fully synchronous, you can simply call _bucket.Upsert(...)
            var bulkResult = await Task.Run(() => _bucket.Upsert(bulkData));

            var successCount = bulkResult.Where(r => r.Value.Success).Count();
            Console.WriteLine("Upserted {0} values", bulkResult.Count);

            // Option 1b: Specify ParallelOptions to customize how the client parallelizes the upserts
            var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = 32 };
            var bulkResult2 = await Task.Run(() => _bucket.Upsert(bulkData, parallelOptions));

            successCount = bulkResult2.Where(r => r.Value.Success).Count();
            Console.WriteLine("Upserted {0} values", successCount);

            // Option 2: Spawn multiple Upsert tasks and wait on for all to complete
            var bulkTasks = bulkData.Select(d => _bucket.UpsertAsync(d.Key, d.Value));
            var bulkResults = await Task.WhenAll(bulkTasks);

            successCount = bulkResults.Where(r => r.Success).Count();
            Console.WriteLine("Upserted {0} values", successCount);
        }

 public void MaxDegreeOfParallelismPropertyBehavior()
 {
     var po = new ParallelOptions {MaxDegreeOfParallelism = 10};
     Assert.That(po.MaxDegreeOfParallelism, Is.EqualTo(10));
     po.MaxDegreeOfParallelism = 3;
     Assert.That(po.MaxDegreeOfParallelism, Is.EqualTo(3));
 }

        public void Read_Streams()
        {
            long values = 0;
            int streams = 0;
            var watch = Stopwatch.StartNew();

            var opt = new ParallelOptions { MaxDegreeOfParallelism = 4 };

            Parallel.ForEach(GetStreams(), opt, stream =>
            //foreach (var stream in GetStreams())
            {
                streams++;
                var repository = BuildRepository(stream);
                foreach (var value in repository.GetValues())
                    values++;

                var speed = values / watch.Elapsed.TotalSeconds;
                Console.WriteLine("Completed {0} number {1} : {2:0} total of {3} ", stream, streams, speed, values);
            }
            );

            watch.Stop();

            Console.WriteLine("read {0} values in {1} streams in {2}", values, streams, watch.Elapsed);
        }

        private static void DownloadProjectsFromGitHub(IEnumerable<Repository> searchRepositoryResult,
            string buildFolder)
        {
            var parallelOptions = new ParallelOptions {MaxDegreeOfParallelism = 1};
            Parallel.ForEach(searchRepositoryResult, parallelOptions, r =>
            {
                var archivePath = Path.Combine(buildFolder, r.Name + ".zip");
                var destinationDirectoryName = Path.Combine(buildFolder, r.Name);
                if (!File.Exists(archivePath))
                {
                    var downloadUrl = r.HtmlUrl + "/archive/master.zip";
                    Console.WriteLine("Downloading from: {0}", downloadUrl);
                    using (var client = new WebClient())
                    {
                        client.DownloadFile(downloadUrl, archivePath);
                    }
                }

                try
                {
                    Console.WriteLine("Extracting to: {0}", destinationDirectoryName);
                    ZipFile.ExtractToDirectory(archivePath, destinationDirectoryName);
                }
                catch (Exception e)
                {
                    Console.WriteLine(e);
                    Directory.Delete(destinationDirectoryName, true);
                }
            });
        }

        public override async Task ExecuteAsync()
        {
            // Call BulkInsert to generate some data
            await new BulkInsert().ExecuteAsync();

            // Generate the keys to retrieve
            var keys = Enumerable.Range(1, 100).Select(i => "dotnetDevguideExample-" + i).ToList();


            // Option 1a: use Get with a IList<string> of document keys to retrieve
            // Note: There is no GetAsync overload that takes multiple keys (yet)
            // which is why this example wraps the synchronized method in a new Task.
            // If your code is fully synchronous, you can simply call _bucket.Get(...)
            var bulkResult = await Task.Run(() => _bucket.Get<Data>(keys));

            var successCount = bulkResult.Where(r => r.Value.Success).Count();
            Console.WriteLine("Got {0} values", bulkResult.Count);

            // Option 1b: Specify ParallelOptions to customize how the client parallelizes the gets
            var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = 32 };
            var bulkResult2 = await Task.Run(() => _bucket.Get<Data>(keys, parallelOptions));

            successCount = bulkResult2.Where(r => r.Value.Success).Count();
            Console.WriteLine("Got {0} values", successCount);

            // Option 2: Spawn multiple Upsert tasks and wait on for all to complete
            var bulkTasks = keys.Select(key => _bucket.GetAsync<Data>(key));
            var bulkResults = await Task.WhenAll(bulkTasks);

            successCount = bulkResults.Where(r => r.Success).Count();
            Console.WriteLine("Got {0} values", successCount);

        }

        public void ValidateRowKeysPairAnsiGetInstanceFromPairAnsi()
        {
            ParallelOptions parOptObj = new ParallelOptions();

            UOPair<char> uoPairObj = new UOPair<char>('?', '?');
            DensePairAnsi dpaObj =
                DensePairAnsi.CreateEmptyInstance(
                new string[] { "R0", "R1", "R2" },
                new string[] { "C0", "C1", "C2", "C3" },
                uoPairObj);

            dpaObj.WriteDensePairAnsi(Constants.FastQTempTxtFileName,
                parOptObj);

            using (RowKeysPairAnsi rkaObj =
                 RowKeysPairAnsi.GetInstanceFromPairAnsi(
                 Constants.FastQTempTxtFileName, parOptObj))
            {
                Assert.AreEqual(4, rkaObj.ColCount);
                Assert.AreEqual(3, rkaObj.RowCount);
            }

            if (File.Exists(Constants.FastQTempTxtFileName))
                File.Delete(Constants.FastQTempTxtFileName);

            ApplicationLog.WriteLine(
                "RowKeysPairAnsi BVT : Validation of GetInstanceFromPairAnsi() method successful");
        }

        public void ValidateRowKeysPDGetInstanceFromPaddedDoubleFileAccess()
        {
            DenseMatrix<string, string, double> denseMatObj =
                GetDenseMatrix();
            ParallelOptions parOptObj = new ParallelOptions();

            denseMatObj.WritePaddedDouble(Constants.FastQTempTxtFileName,
                parOptObj);

            using (RowKeysPaddedDouble rkpdObj =
                RowKeysPaddedDouble.GetInstanceFromPaddedDouble(
                Constants.FastQTempTxtFileName, parOptObj, FileAccess.ReadWrite,
                FileShare.ReadWrite))
            {
                Assert.AreEqual(denseMatObj.ColCount, rkpdObj.ColCount);
                Assert.AreEqual(denseMatObj.RowCount, rkpdObj.RowCount);
                Assert.AreEqual(denseMatObj.RowKeys.Count, rkpdObj.RowKeys.Count);
                Assert.AreEqual(denseMatObj.ColKeys.Count, rkpdObj.ColKeys.Count);
            }

            if (File.Exists(Constants.FastQTempTxtFileName))
                File.Delete(Constants.FastQTempTxtFileName);

            ApplicationLog.WriteLine(
                "RowKeysPaddedDouble BVT : Validation of GetInstanceFromPaddedDouble(file-access) method successful");
        }

        public void ValidateRowKeysAnsiGetInstanceFromDenseAnsi()
        {
            DenseMatrix<string, string, double> denseMatObj =
                GetDenseMatrix();
            ParallelOptions parOptObj = new ParallelOptions();

            denseMatObj.WriteDenseAnsi(Constants.FastQTempTxtFileName,
                parOptObj);

            using (RowKeysAnsi rkaObj =
                RowKeysAnsi.GetInstanceFromDenseAnsi(Constants.FastQTempTxtFileName,
                parOptObj))
            {

                Assert.AreEqual(denseMatObj.ColCount, rkaObj.ColCount);
                Assert.AreEqual(denseMatObj.RowCount, rkaObj.RowCount);
                Assert.AreEqual(denseMatObj.RowKeys.Count, rkaObj.RowKeys.Count);
                Assert.AreEqual(denseMatObj.ColKeys.Count, rkaObj.ColKeys.Count);
            }

            if (File.Exists(Constants.FastQTempTxtFileName))
                File.Delete(Constants.FastQTempTxtFileName);

            ApplicationLog.WriteLine(
                "RowKeysAnsi BVT : Validation of GetInstanceFromDenseAnsi() method successful");
        }

 public static BitmapSource GetColorBrushImage(Color color)
 {
     var stream = new MemoryStream();
     Resource.EmptyBrush.Save(stream, ImageFormat.Png);
     stream.Seek(0, SeekOrigin.Begin);
     var bitmap = CreateBitmapImage(stream);
     int width = bitmap.PixelWidth;
     int height = bitmap.PixelHeight;
     var resultBitmap = new WriteableBitmap(width, height, 92, 92, PixelFormats.Bgra32, bitmap.Palette);
     var imageBytes = bitmap.GetBytes();
     DisposeImage(bitmap);
     var po = new ParallelOptions {MaxDegreeOfParallelism = 4};
     Parallel.For(0, width*height, po, i =>
     {
         int index = i*4;
         if (imageBytes[index + 3] < 16)
         {
             imageBytes[index] = color.B;
             imageBytes[index + 1] = color.G;
             imageBytes[index + 2] = color.R;
             imageBytes[index + 3] = color.A;
         }
     });
     resultBitmap.WritePixels(new Int32Rect(0, 0, width,  height), imageBytes, 4*width, 0);
     if (resultBitmap.CanFreeze)
     {
         resultBitmap.Freeze();
     }
     return resultBitmap;
 }

        public virtual  void Initialize(string name, XmlElement xmlConfig)
        {
            _name = name;
            _populationSize = XmlUtils.GetValueAsInt(xmlConfig, "PopulationSize");
            _specieCount = XmlUtils.GetValueAsInt(xmlConfig, "SpecieCount");
            _activationScheme = ExperimentUtils.CreateActivationScheme(xmlConfig, "Activation");
            _complexityRegulationStr = XmlUtils.TryGetValueAsString(xmlConfig,
                "DefaultComplexityRegulationStrategy");
            _complexityThreshold = XmlUtils.TryGetValueAsInt(xmlConfig, "ComplexityThreshold");
            _description = XmlUtils.TryGetValueAsString(xmlConfig, "Description");
            _parallelOptions = ExperimentUtils.ReadParallelOptions(xmlConfig);

            _eaParams = new NeatEvolutionAlgorithmParameters();
            _eaParams.SpecieCount = _specieCount;
            _neatGenomeParams = new NeatGenomeParameters();
            _neatGenomeParams.FeedforwardOnly = _activationScheme.AcyclicNetwork;

            DefaultComplexityRegulationStrategy = ExperimentUtils.CreateComplexityRegulationStrategy(
                _complexityRegulationStr,
                _complexityThreshold);
            DefaultSpeciationStrategy = new KMeansClusteringStrategy<NeatGenome>(new ManhattanDistanceMetric(1.0, 0.0, 10.0));
            DefaultNeatEvolutionAlgorithm = new NeatEvolutionAlgorithm<NeatGenome>(
                    NeatEvolutionAlgorithmParameters,
                    DefaultSpeciationStrategy,
                    DefaultComplexityRegulationStrategy);
        }

        private static void DownloadInParallel(IEnumerable<Show> showsToDownload)
        {
            var po = new ParallelOptions {MaxDegreeOfParallelism = MaxSimultaneousDownloads};
            Parallel.ForEach(showsToDownload, po, show =>
                {
                    try
                    {
                        using (var httpClient = new HttpClient())
                        {
                            var downloadStream = httpClient.GetStreamAsync(show.Mp3Uri).Result;

                            var file = new FileInfo(_saveDirectory + string.Format(@"\Hanselminutes_{0}.mp3", show.Id));
                            using (downloadStream)
                            using (var fileStream = file.Create())
                            {
                                Console.WriteLine(string.Format("Downloading show {0}", show.Id));
                                downloadStream.CopyTo(fileStream);
                            }

                            Console.WriteLine(string.Format("Show {0} downloaded to {1}", show.Id, file));
                        }
                    }
                    catch (Exception e)
                    {
                        Console.Error.WriteLine(e);
                    }
                });
        }

        public void Test_Timed_Execution_Parallel_Client()
        {
            var options = new ParallelOptions { MaxDegreeOfParallelism = 4 };
            var n = 1000;//set to a higher # if needed

            using (var cluster = new Cluster("couchbaseClients/couchbase"))
            {
                using (var bucket = cluster.OpenBucket())
                {
                    using (new OperationTimer())
                    {
                        var temp = bucket;
                        Parallel.For(0, n, options, i =>
                        {
                            var key = string.Format("key{0}", i);
                            var value = (int?) i;
                            var result = temp.Upsert(key, value);
                            Assert.IsTrue(result.Success);

                            var result1 = temp.Get<int?>(key);
                            Assert.IsTrue(result1.Success);
                            Assert.AreEqual(i, result1.Value);
                        });
                    }
                }
            }
        }

        public void ValidateMatrixFactoryParse()
        {
            denseMatObj = GetDenseMatrix();

            MatrixFactory<String, String, Double> mfObj =
                MatrixFactory<String, String, Double>.GetInstance();

            ParallelOptions poObj = new ParallelOptions();

            TryParseMatrixDelegate<string, string, double> a =
                new TryParseMatrixDelegate<string, string, double>(this.TryParseMatrix);
            mfObj.RegisterMatrixParser(a);
            // Writes the text file
            denseMatObj.WritePaddedDouble(Constants.FastQTempTxtFileName, poObj);

            Matrix<string, string, double> newMatObj =
                mfObj.Parse(Constants.FastQTempTxtFileName, double.NaN, poObj);

            Assert.AreEqual(denseMatObj.RowCount, newMatObj.RowCount);
            Assert.AreEqual(denseMatObj.RowKeys.Count, newMatObj.RowKeys.Count);
            Assert.AreEqual(denseMatObj.ColCount, newMatObj.ColCount);
            Assert.AreEqual(denseMatObj.ColKeys.Count, newMatObj.ColKeys.Count);
            Assert.AreEqual(denseMatObj.Values.Count(), newMatObj.Values.Count());

            ApplicationLog.WriteLine(
                "MatrixFactory BVT : Successfully validated Parse() method");
        }