public void SparseArraySimpleTest()
        {
            var stringArrayRef = new string[1000];
            var stringArray = new SparseArray<string>(1000);

            var randomGenerator = new RandomGenerator(66707770); // make this deterministic
            for (int idx = 0; idx < 1000; idx++)
            {
                if (randomGenerator.Generate(2.0) > 1)
                { // add data.
                    stringArrayRef[idx] = idx.ToString();
                    stringArray[idx] = idx.ToString();
                }
                else
                {
                    stringArrayRef[idx] = null;
                    stringArray[idx] = null;
                }
            }

            for (int idx = 0; idx < 1000; idx++)
            {
                Assert.AreEqual(stringArrayRef[idx], stringArray[idx]);
            }
        }

        public void Generate(SparseArray<float> production, SparseArray<float> attractions)
        {
            var ages = this.Root.Demographics.AgeRates;
            var studentRates = this.Root.Demographics.SchoolRates.GetFlatData();
            var zones = this.Root.ZoneSystem.ZoneArray.GetFlatData();
            var prod = production.GetFlatData();

            for ( int i = 0; i < zones.Length; i++ )
            {
                // this is only null for externals
                var studentRatesForZone = studentRates[i];
                if ( studentRatesForZone == null )
                {
                    // if it is an external ignore
                    prod[i] = 0f;
                }
                else
                {
                    // otherwise compute the production
                    var pd = zones[i].PlanningDistrict;
                    prod[i] = zones[i].Population * ages[zones[i].ZoneNumber, this.Age] * studentRatesForZone[this.Age, 0] *
                        StudentDailyRates[pd, 0, this.Age] * StudentTimeOfDayRates[pd, 0, this.Age];
                }
            }

            SaveProductionToDisk( zones, prod );
        }

        public override void Generate(SparseArray<float> production, SparseArray<float> attractions)
        {
            if(LoadData)
            {
                LoadExternalWorkerRates.LoadData();
                LoadWorkAtHomeRates.LoadData();
                LoadExternalJobsRates.LoadData();
                ExternalRates = LoadExternalWorkerRates.GiveData();
                WorkAtHomeRates = LoadWorkAtHomeRates.GiveData();
                ExternalRates = LoadExternalJobsRates.GiveData();
            }
            var flatProduction = production.GetFlatData();
            var flatWah = new float[flatProduction.Length];
            var totalProduction = ComputeProduction(flatProduction, flatWah);
            var totalAttraction = ComputeAttraction(attractions.GetFlatData());

            Normalize(production.GetFlatData(), attractions.GetFlatData(), totalProduction, totalAttraction);
            totalAttraction = RemoveWAHFromAttraction(attractions.GetFlatData(), flatWah);
            StoreProductionData(production);
            WriteGenerationFile(totalProduction, totalAttraction);
            WriteAttractionFile(attractions);
            if(LoadData)
            {
                LoadExternalWorkerRates.UnloadData();
                LoadWorkAtHomeRates.UnloadData();
                LoadExternalJobsRates.UnloadData();
                WorkAtHomeRates = null;
                ExternalRates = null;
                ExternalJobs = null;
            }
        }

        public override void Generate(SparseArray<float> production, SparseArray<float> attractions)
        {
            if ( LoadData )
            {
                if ( DailyRates == null )
                {
                    this.LoadDailyRates.LoadData();
                    this.DailyRates = this.LoadDailyRates.GiveData();
                }
                if ( TimeOfDayRates == null )
                {
                    this.LoadTimeOfDayRates.LoadData();
                    this.TimeOfDayRates = this.LoadTimeOfDayRates.GiveData();
                }
            }
            var flatProduction = production.GetFlatData();
            var numberOfIndexes = flatProduction.Length;

            // Compute the Production
            ComputeProduction( flatProduction, numberOfIndexes );
            float totalProduction = flatProduction.Sum();
            WriteGenerationCSV( totalProduction );
            //The PoRPoS Model does NOT include having an attraction component.  The distribution will handle this case.
            if ( LoadData )
            {
                this.DailyRates = null;
                this.TimeOfDayRates = null;
            }
        }

        public override void Run()
        {
            var numberOfCategories = this.Categories.Count;
            SparseArray<float>[] O = new SparseArray<float>[numberOfCategories];
            SparseArray<float>[] D = new SparseArray<float>[numberOfCategories];

            Generation = true;
            for ( int i = 0; i < numberOfCategories; i++ )
            {
                O[i] = Root.ZoneSystem.ZoneArray.CreateSimilarArray<float>();
                D[i] = Root.ZoneSystem.ZoneArray.CreateSimilarArray<float>();
                this.Categories[i].Generate( O[i], D[i] );
            }
            Generation = false;

            var modeSplit = this.ModeSplit.ModeSplit( this.Distribution.Distribute( O, D, this.Categories ), this.Categories.Count );
            if ( this.Transpose )
            {
                TransposeMatrix( modeSplit );
            }
            if ( this.SaveOutput )
            {
                if ( !Directory.Exists( this.PurposeName ) )
                {
                    Directory.CreateDirectory( this.PurposeName );
                }
                for ( int i = 0; i < modeSplit.Count; i++ )
                {
                    this.WriteModeSplit( modeSplit[i], this.Root.Modes[i], this.PurposeName );
                }
            }
            this.Flows = modeSplit;
        }

 public override void Run()
 {
     if ( !this.Execute ) return;
     // we actually don't write our mode choice
     var numberOfCategories = this.Categories.Count;
     SparseArray<float>[] O = new SparseArray<float>[numberOfCategories];
     SparseArray<float>[] D = new SparseArray<float>[numberOfCategories];
     for ( int i = 0; i < O.Length; i++ )
     {
         O[i] = Root.ZoneSystem.ZoneArray.CreateSimilarArray<float>();
         D[i] = Root.ZoneSystem.ZoneArray.CreateSimilarArray<float>();
         this.Categories[i].Generate( O[i], D[i] );
     }
     // if we only need to run generation we are done
     if ( this.OnlyDoGeneration ) return;
     // we don't do mode choice
     foreach ( var distributionData in this.Distribution.Distribute( O, D, this.Categories ) )
     {
         var interative = this.ModeSplit as IInteractiveModeSplit;
         if ( interative != null )
         {
             interative.EndInterativeModeSplit();
         }
         if ( !String.IsNullOrWhiteSpace( this.SaveResultFileName ) )
         {
             SaveFriction( distributionData.GetFlatData() );
         }
     }
 }

 public void Generate(SparseArray<float> production, SparseArray<float> attractions)
 {
     float totalProduction = 0;
     float totalAttraction = 0;
     foreach (var zone in production.ValidIndexies())
     {
         float prod, attr;
         production[zone] = (prod = this.PopulationFactor * production[zone]);
         attractions[zone] = (attr = this.EmploymentFactor * attractions[zone]);
         totalProduction += prod;
         totalAttraction += attr;
     }
     if (totalAttraction <= 0)
     {
         throw new XTMF.XTMFRuntimeException("There is no employment in the zone system!");
     }
     else if (totalProduction <= 0)
     {
         throw new XTMF.XTMFRuntimeException("There is no population in the zone system!");
     }
     // Normalize the attractions
     var inverseTotalAttraction = 1 / totalAttraction; // inverse totalAttraction to save on divisions
     foreach (var zone in Root.ZoneSystem.ZoneArray.ValidIndexies())
     {
         attractions[zone] = (attractions[zone] * inverseTotalAttraction) * totalProduction;
     }
 }

        public override void Generate(SparseArray<float> production, SparseArray<float> attractions)
        {
            if ( LoadData )
            {
                if ( DailyRates == null )
                {
                    this.LoadDailyRates.LoadData();
                    this.DailyRates = this.LoadDailyRates.GiveData();
                }
                if ( TimeOfDayRates == null )
                {
                    this.LoadTimeOfDayRates.LoadData();
                    this.TimeOfDayRates = this.LoadTimeOfDayRates.GiveData();
                }
            }
            var flatProduction = production.GetFlatData();
            var flatAttraction = attractions.GetFlatData();

            var numberOfIndexes = flatAttraction.Length;

            // Compute the Production and Attractions
            ComputeProduction( flatProduction, flatAttraction, numberOfIndexes );

            //We do not normalize the attraction
            if ( LoadData )
            {
                this.LoadDailyRates.UnloadData();
                this.LoadTimeOfDayRates.UnloadData();
                DailyRates = null;
                TimeOfDayRates = null;
            }
        }

 public void TestCSVODC()
 {
     try
     {
         int[] zones = new int[] { 0, 1, 2, 3, 4, 5, 6 };
         SparseArray<int> referenceArray = new SparseArray<int>( new SparseIndexing()
         {
             Indexes = new SparseSet[]
         {
             new SparseSet() { Start = 0, Stop = 6 }
         }
         } );
         float[][][] allData = new float[1][][];
         var data = CreateData( zones.Length );
         CreateCSVFile( zones, data, "Test.csv" );
         allData[0] = data;
         var writer = new ODMatrixWriter<int>( referenceArray, 1, 1 );
         writer.LoadCSVTimes( "Test.csv", false, 0, 0 );
         writer.Save( "Test.odc", false );
         var odcFloatData = ConvertData( allData, zones.Length, 1, 1 );
         ValidateData( zones, odcFloatData, "Test.odc" );
     }
     finally
     {
         File.Delete( "Test.csv" );
         File.Delete( "Test.odc" );
     }
 }

 public EmmeMatrix(SparseArray<IZone> zoneSystem, float[][] data)
 {
     var zones = zoneSystem.GetFlatData();
     MagicNumber = EmmeMagicNumber;
     Version = 1;
     Type = DataType.Float;
     Dimensions = 2;
     float[] temp = new float[zones.Length * zones.Length];
     Indexes = new int[2][];
     for(int i = 0; i < Indexes.Length; i++)
     {
         var row = Indexes[i] = new int[zones.Length];
         for(int j = 0; j < row.Length; j++)
         {
             row[j] = zones[j].ZoneNumber;
         }
     }
     for(int i = 0; i < data.Length; i++)
     {
         Array.Copy(data[i], 0, temp, i * zones.Length, zones.Length);
     }
     FloatData = temp;
     DoubleData = null;
     SignedIntData = null;
     UnsignedIntData = null;
 }

        private SparseArray<DataRenderer> GetRenderers(IEntity entity) {
            SparseArray<DataRenderer> renderers;

            if (_renderers.TryGetValue(entity.UniqueId, out renderers) == false) {
                renderers = new SparseArray<DataRenderer>();
                _renderers[entity.UniqueId] = renderers;
            }

            return renderers;
        }

        public void SetOneHundredElements()
        {
            SparseArray<int> array = new SparseArray<int>(10);

            for (int k = 0; k < 100; k++)
                array[(ulong)k] = k;

            for (int k = 0; k < 100; k++)
                Assert.AreEqual(k, array[(ulong)k]);
        }

        public void GettingNonExistingReturnsNull()
        {
            var array = new SparseArray<string>();

            Assert.IsNull(array.GetValue(10000));

            // We shouldn't have allocated items.

            Assert.AreEqual("Values=20", array.ToString());
        }

        public void GetNegativeShouldNotFail()
        {
            var array = new SparseArray<string>();

            Assert.IsNull(array.GetValue(-10));

            // We shouldn't have allocated items.

            Assert.AreEqual("Values=20", array.ToString());
        }

        public SparseTwinIndex<float> ProcessFlow(SparseArray<float> O, SparseArray<float> D, int[] validIndexes, SparseArray<float> attractionStar = null)
        {
            int length = validIndexes.Length;
            Productions = O;
            Attractions = D;
            if(attractionStar == null)
            {
                AttractionsStar = D.CreateSimilarArray<float>();
            }
            else
            {
                AttractionsStar = attractionStar;
            }
            FlowMatrix = Productions.CreateSquareTwinArray<float>();
            if(Friction == null)
            {
                InitializeFriction(length);
            }
            var flatAttractionStar = AttractionsStar.GetFlatData();
            float[] oldTotal = new float[flatAttractionStar.Length];
            var flatAttractions = Attractions.GetFlatData();
            for(int i = 0; i < length; i++)
            {
                flatAttractionStar[i] = 1f;
                oldTotal[i] = flatAttractions[i];
            }
            int iteration = 0;
            float[] columnTotals = new float[length];
            var balanced = false;
            do
            {
                if(ProgressCallback != null)
                {
                    // this doesn't go to 100%, but that is alright since when we end, the progress
                    // of the calling model should assume we hit 100%
                    ProgressCallback(iteration / (float)MaxIterations);
                }
                Array.Clear(columnTotals, 0, columnTotals.Length);
                if(Vector.IsHardwareAccelerated)
                {
                    VectorProcessFlow(columnTotals, FlowMatrix.GetFlatData());
                }
                else
                {
                    ProcessFlow(columnTotals);
                }
                balanced = Balance(columnTotals, oldTotal);
            } while((++iteration) < MaxIterations && !balanced);

            if(ProgressCallback != null)
            {
                ProgressCallback(1f);
            }
            return FlowMatrix;
        }

        public void TestSparseArraySimple()
        {
            // intialize.
            var array = new SparseArray<int>(10);

            // fill and resize in the process.
            for (int idx = 0; idx < 1000; idx++)
            {
                if (idx >= array.Length)
                {
                    array.Resize(idx + 100);
                }
                array[idx] = idx;
            }
            for (int idx = 5000; idx < 10000; idx++)
            {
                if (idx >= array.Length)
                {
                    array.Resize(idx + 100);
                }
                array[idx] = idx;
            }

            // test content.
            for (int idx = 0; idx < 1000; idx++)
            {
                Assert.AreEqual(idx, array[idx]);
            }
            for (int idx = 1001; idx < 5000; idx++)
            {
                Assert.AreEqual(0, array[idx]);
            }
            for (int idx = 5000; idx < 10000; idx++)
            {
                Assert.AreEqual(idx, array[idx]);
            }

            // test enumerator.
            var list = new List<int>(array);
            for (int idx = 0; idx < 1000; idx++)
            {
                Assert.AreEqual(idx, list[idx]);
            }
            for (int idx = 1001; idx < 5000; idx++)
            {
                Assert.AreEqual(0, list[idx]);
            }
            for (int idx = 5000; idx < 10000; idx++)
            {
                Assert.AreEqual(idx, list[idx]);
            }
        }

        public void SetOneThousandSparseElements()
        {
            SparseArray<int> array = new SparseArray<int>(10);

            for (int k = 0; k < 1000; k++)
            {
                array[(ulong)(k*1000)] = k;
                Assert.AreEqual(k, array[(ulong)(k*1000)]);
            }

            for (int k = 0; k < 1000; k++)
                Assert.AreEqual(k, array[(ulong)(k*1000)]);
        }

        public static SparseTwinIndex<float> Process(SparseArray<float> production, float[] friction)
        {
            var ret = production.CreateSquareTwinArray<float>();
            var flatRet = ret.GetFlatData();
            var flatProduction = production.GetFlatData();
            var numberOfZones = flatProduction.Length;
            try
            {
                // Make all of the frictions to the power of E
                Parallel.For( 0, friction.Length, new ParallelOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount },
                    delegate(int i)
                    {
                        friction[i] = (float)Math.Exp( friction[i] );
                    } );

                Parallel.For( 0, numberOfZones, new ParallelOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount },
                    delegate(int i)
                    {
                        float sum = 0f;
                        var iIndex = i * numberOfZones;
                        // gather the sum of the friction
                        for ( int j = 0; j < numberOfZones; j++ )
                        {
                            sum += friction[iIndex + j];
                        }
                        if ( sum <= 0 )
                        {
                            return;
                        }
                        sum = 1f / sum;
                        for ( int j = 0; j < numberOfZones; j++ )
                        {
                            flatRet[i][j] = flatProduction[i] * ( friction[iIndex + j] * sum );
                        }
                    } );
            }
            catch ( AggregateException e )
            {
                if ( e.InnerException is XTMFRuntimeException )
                {
                    throw new XTMFRuntimeException( e.InnerException.Message );
                }
                else
                {
                    throw new XTMFRuntimeException( e.InnerException.Message + "\r\n" + e.InnerException.StackTrace );
                }
            }
            return ret;
        }

        public void SetOneThousandElements()
        {
            SparseArray<int> array = new SparseArray<int>(10);

            for (int k = 0; k < 1000; k++)
            {
                array[(ulong)k] = k;
                Assert.AreEqual(k, array[(ulong)k]);
                if (k>1)
                    Assert.AreEqual(1, array[1], string.Format("One lost at {0}", k));
            }

            for (int k = 0; k < 1000; k++)
                Assert.AreEqual(k, array[(ulong)k]);
        }

        public ArrayObject(Environment env, uint length)
            : base(env, env.Maps.Array, env.Prototypes.Array)
        {
            this.length = length;

            if (length > 0x20000)
            {
                this.sparse = new SparseArray();
                this.isDense = false;
            }
            else
            {
                this.dense = new Descriptor[length];
                this.isDense = true;
            }
        }