public void Create(ConstructGen<double> con_)
    {
      setupDataTable();

      double cumulative = 0d;

      foreach (var date in con_.Dates)
      {
        var values = con_.GetValues(date);

        var wt = values[(int)Headings.Weight];

        var dailyPnl = values[(int)Headings.Pnl];

        cumulative += dailyPnl;

        dt.LoadDataRow(new object[] 
        {
          date,
          cumulative,
          values[(int)Headings.Spot],
          wt > 0d ? wt : 0d,
          wt < 0d ? wt : 0d
          }, true);
      }

      m_con = con_;
    }

    public void Refresh()
    {
      // going to get historical values from our database
      ConstructGen<double> hist = new ConstructGen<double>(m_tickers.Count<string>());

      for (int i = 0; i < m_tickers.Length; ++i)
      {
        var histvalues = BbgTalk.HistoryRequester.GetHistory(SI.Data.DataConstants.DATA_START, m_tickers[i],
          "PX_LAST", false, null);

        //histvalues = histvalues.unwind_1d();

        hist.SetColumnValues(i, histvalues);
      }

      hist.SortKeys();

      // initialise 'today's' values to previous day
      hist.SetValues(DateTime.Today, (double[])hist.GetValues(hist.LastDate).Clone());

      double[] avgs = new double[m_windowLength];

      // initialise the avgs array NOTE: today's value is in item with index 0
      for (int i = 0; i < m_windowLength; ++i)
        avgs[i] = hist.GetValues(hist.Dates[hist.Dates.Count - 1 - i]).Average();

      m_avgs = avgs;
      m_liveValues = hist.GetValues(DateTime.Today);
    }

    public override void DoWeights(ConstructGen<double> signalResults_, ConstructGen<double> fullSignalResults_, ConstructGen<double> wts_, List<ProductBase> products_, ConstructGen<double> filters_)
    {
      List<SortHelpClass> list = new List<SortHelpClass>();

      foreach (DateTime date in signalResults_.Dates)
      {
        //wts_.MultValue(date, i, val, val < 0);

        list.Clear();

        for (int i = 0; i < signalResults_.ArrayLength; ++i)
        {
          double val = signalResults_.GetValue(date, i);

          if (double.IsNaN(val) == false)
            list.Add(new SortHelpClass(val, i, m_abs,false,false));
        }

        QuickSort.Sort<SortHelpClass>(list);



        for (int i = 0; i < wts_.ArrayLength && i < list.Count; ++i)
        {
          wts_.MultValue(date, list[i].ArrayIndex, list[i].Value, list[i].Value < 0);
        }

      }

    }

    public override void DoWeights(ConstructGen<double> signalResults_, ConstructGen<double> fullSignalResults_, ConstructGen<double> wts_, List<ProductBase> products_, ConstructGen<double> filters_)
    {
      double[] sigToday = new double[products_.Count];

      foreach (DateTime date in signalResults_.Dates)
      {
        double[] filterToday = filters_.GetValues(date);
        int productValidCount = 0;
        double sum = 0;

        for (int i = 0; i < sigToday.Length; ++i)
        {
          sigToday[i] = signalResults_.GetValue(date, i);
          if (double.IsNaN(filterToday[i]) == false && double.IsNaN(sigToday[i]) == false)
          {
            ++productValidCount;
            sum += sigToday[i];
          }
        }

        var avg = sum / Convert.ToDouble(productValidCount);

        for (int i = 0; i < sigToday.Length; ++i)
        {
          if (double.IsNaN(filterToday[i]) == false && double.IsNaN(sigToday[i]) == false)
            wts_.MultValue(date, i, sigToday[i] - avg);
        }
      }
    }

    public static ConstructGen<double> DoScaleWeights(ConstructGen<double> wts_, TraderArgs args_, Func<DateTime,double> getVolTargetForDate_)
    {
      if (wts_.ColumnHeadings == null)
        wts_.ColumnHeadings = args_.Products.Select(x => x.Name).ToArray();

      var logReturns = args_.AllProductPrices(fillInGapsWithPrevious_: true).ToLogReturns(
        args_.Products.Select(x => x.Convention).ToArray());

      var scaledWts = new ConstructGen<double>(wts_.ColumnHeadings);

      foreach (var date in wts_.Dates)
      {
        var wtArr = wts_.GetValues(date);

        for (int i = 0; i < wtArr.Length; ++i)
          if (double.IsInfinity(wtArr[i]))
            wtArr[i] = 0d;

        int indexOfDate = logReturns.FindIndexOfEffectiveDateOnDate(date);

        indexOfDate = (indexOfDate < 252) ? 252 : indexOfDate - 1; // note offset

        var subValues = logReturns.GetSubValues(logReturns.Dates[indexOfDate - 251], logReturns.Dates[indexOfDate]);

        var covArr = SI.Data.FXHistCovar.MatCovar(subValues.ToArray());

        var cov = new CovarianceItem(covArr);

        scaledWts.SetValues(date, cov.ScaleSeries(wtArr, getVolTargetForDate_(date)));
      }

      return scaledWts;
    }

    public override void DoWeights(ConstructGen<double> signalResults_, ConstructGen<double> fullSignalResults_, ConstructGen<double> wts_, List<ProductBase> products_, ConstructGen<double> filters_)
    {
      for (int i = 0; i < products_.Count; ++i)
      {
        double avg = 0;
        for (int j = 0; j < signalResults_.Dates.Count; ++j)
        {
          if(j<WindowLength)
          {
            wts_.SetValue(signalResults_.Dates[j],i,0);
            continue;
          }

          // if the product is not valid on this date then the weight is zero.
          if (double.IsNaN(filters_.GetValue(signalResults_.Dates[j],i)))
          {
            wts_.SetValue(signalResults_.Dates[j], i, 0.0);
          }
          else
          {
            for (int y = 0; y < WindowLength; ++y)
              avg += signalResults_.GetValue(signalResults_.Dates[j - y], i);
            avg /= WindowLength;

            double val = signalResults_.GetValue(signalResults_.Dates[j], i) / avg;

            wts_.MultValue(signalResults_.Dates[j], i, val, false);
          }
        }
      }
    }

    public static ConstructGen<double> GetSmoothCurvesColumnsAreCurvePoints(DateTime valueDate_, uint curveCount_, BondMarket market_, BondField field_, SI.Data.BondCurves curve_, string close_ = "MLP", string source_ = "MLP")
    {
      DateTime date = valueDate_;

      var points = new List<decimal>();

      for (decimal d = 1M; d < 30M; d = d + 0.25M)
        points.Add(d);

      var con = new ConstructGen<double>(points.Select(x => x.ToString()).ToArray());

      for (int i = 0; i < curveCount_; ++i)
      {
        var curve = GetSmoothCurve(date, market_, field_, curve_, close_, source_);
        if (curve == null) continue;

        foreach (var node in curve.GetNodes())
        {
          int index = points.IndexOf(node);

          if (index == -1) continue;

          var point = curve.GetValue(node);

          if(point.HasValue)
            con.SetValue(date, index, point.Value);
        }

        date = MyCalendar.PrevWeekDay(date);
      }

      con.SortKeys();

      return con;
    }

        /// <summary>
        /// This is to compute the daily SS before each event
        /// </summary>
        public static Tuple<double, double> ComputeDailySSBeforeEachEvent(ConstructGen<double> seasonalitySeries, int eventDateIndex, int ssWindow)
        {
            // inputs: data series
            // now - event return / std
            var beforeEventSSSeries = new List<double>();
            
            foreach (var eventDate in seasonalitySeries.Dates.OrderBy(d => d))
            {
                var b4Event = seasonalitySeries[eventDate].Where((x, i) => !double.IsNaN(x) && i < eventDateIndex && Math.Abs(eventDateIndex - i) <= ssWindow);
                if (b4Event != null)
                {
                    b4Event.ForEach(e => beforeEventSSSeries.Add(e));                    
                }
            }

            double avgSS = Double.NaN;
            double RSS = double.NaN;
            if (beforeEventSSSeries.Any())
            {
                var validSeries = beforeEventSSSeries.Where(s => !double.IsNaN(s)).ToArray();
                var stddev = Statistics.Stdev(validSeries);
                avgSS = validSeries.Select(s => s/stddev).Average();
                var recent20PercentSS = validSeries.Select(s => s/stddev)
                    .Skip((int) (validSeries.Length*0.8)).Average();

                RSS = recent20PercentSS - avgSS;
            }
            return new Tuple<double, double>(avgSS, RSS);
        }

 public void Setup(ConstructGen<double> input_, TZ tz_)
 {
   if (input_ == null)
     _cache[tz_] = new DateTime[0];
   else
     _cache[tz_] = m_builderFunction(input_);
 }

    protected override ReturnsEval.DataSeriesEvaluator doPnl(TraderArgs args_, ConstructGen<double> wts_)
    {
      var priceReturns =
        args_.AllProductPrices(fillInGapsWithPrevious_: true)
          .ToReturns(args_.Products.Select(x => x.Convention).ToArray());


      var stratReturns = new ConstructGen<double>(priceReturns.ColumnHeadings);

      double[] appliedWeights = null;

      for (int i = 0; i < priceReturns.Dates.Count; ++i)
      {
        var date = priceReturns.Dates[i];

        var priceReturnsArr = priceReturns.GetValues(date);

        if (appliedWeights != null)
        {
          for (int j = 0; j < priceReturnsArr.Length; ++j)
            stratReturns.SetValue(date, j, appliedWeights[j]*priceReturnsArr[j]);
        }

        if (wts_.Dates.Contains(date))
        {
          appliedWeights = wts_.GetValues(date);
        }
      }

      var eval = new ReturnsEval.DataSeriesEvaluator("Gen pnl from weights", ReturnsEval.DataSeriesType.Returns);
      eval.AddInnerSeries(stratReturns.Dates.ToArray(), stratReturns.ToArray(), stratReturns.ColumnHeadings);

      return eval;
    }

    protected ATMVolsRankGroup(FXGroup group_)
    {
      var currencies = Singleton<FXIDs>.Instance.Where(x => x.IsGroup(group_)).ToArray();

      var oneWeek = new ConstructGen<double>(currencies.Select(x => x.Code).ToArray());
      var oneMonth = new ConstructGen<double>(currencies.Select(x => x.Code).ToArray());

      for (int i = 0; i < currencies.Length; ++i)
      {
        {
          var vols = BbgTalk.HistoryRequester.GetHistory(DataConstants.DATA_START, currencies[i].AtTheMoneyVolTicker_1W, "PX_LAST", false);
          oneWeek.SetColumnValues(i, vols);
        }
        {
          var vols = BbgTalk.HistoryRequester.GetHistory(DataConstants.DATA_START, currencies[i].AtTheMoneyVolTicker_1M, "PX_LAST", false);
          oneMonth.SetColumnValues(i, vols);
        }
      }

      {
        oneWeek.SortKeys();
        var avg1W = oneWeek.AvgRows();
        ATM_1W_Avg = avg1W;
        ATM_1W_o6M = avg1W.ToPercentileRanked(126);
        ATM_1W_o1Y = avg1W.ToPercentileRanked(252);
      }

      {
        oneMonth.SortKeys();
        var avg1M = oneMonth.AvgRows();
        ATM_1M_Avg = avg1M;
        ATM_1M_o6M = avg1M.ToPercentileRanked(126);
        ATM_1M_o1Y = avg1M.ToPercentileRanked(252);
      }
    }

    public static ConstructGen<double> TestOnCT()
    {
      var str = KnownStructure.US_WN;

      var comps = new[]
      {
        new ChartComponent(),
        new ChartComponent(),
        new ChartComponent(),
      };

      var bondsRollMonthly = Generator.Go(cashBondRollOption_: Generator.CashBondRolling.Monthly);
      var spreadsRollMonthly = new Dictionary<BondCurves, List<CTDLine<BondSpreads>>>();
      spreadsRollMonthly[BondCurves.USD3mLibor] = Generator.GetSpreads(bondsRollMonthly, BondCurves.USD3mLibor, Generator.CashBondRolling.Monthly);

      var cdRollMonthly = new ChartDataSource(comps, spreadsRollMonthly, bondsRollMonthly);

      var impl = StructureFactory.GetStructure(str);

      impl.Configure(comps);

      var data = cdRollMonthly.GetData().SmoothedData.GetEndValues(650);

      var con = new ConstructGen<double>(1);
      con.SetColumnValues(0, data);

      return con;
    }

    public void DoWeights(ConstructGen<double> signalResults_, ConstructGen<double> fullSignalResults_, ConstructGen<double> wts_, List<ProductBase> products_, ConstructGen<double> filters_)
    {
      if (ValueWt.Enabled)
        ValueWt.DoWeights(signalResults_, fullSignalResults_, wts_, products_,filters_);

      if (VolWt.Enabled)
        VolWt.DoWeights(signalResults_, fullSignalResults_, wts_, products_, filters_);

      if (Rank.Enabled)
        Rank.DoWeights(signalResults_, fullSignalResults_, wts_, products_, filters_);

      if (Normalise.Enabled)
        Normalise.DoWeights(signalResults_, fullSignalResults_, wts_, products_, filters_);

      if (Demeaning.Enabled)
        Demeaning.DoWeights(signalResults_, fullSignalResults_, wts_, products_, filters_);

      if (CrossSectionalDemeaning.Enabled)
        CrossSectionalDemeaning.DoWeights(signalResults_, fullSignalResults_, wts_, products_, filters_);

      if (NormsDist.Enabled)
        NormsDist.DoWeights(signalResults_, fullSignalResults_, wts_, products_, filters_);

      if (Sign.Enabled)
        Sign.DoWeights(signalResults_, fullSignalResults_, wts_, products_, filters_);

      if (Side.Enabled)
        Side.DoWeights(signalResults_, fullSignalResults_, wts_, products_, filters_);

      if (Benjo.Enabled)
        Benjo.DoWeights(signalResults_, fullSignalResults_, wts_, products_, filters_);
    }

    private ConstructGen<double> getConstructOfInvoiceSpreads()
    {
      var con = new ConstructGen<double>(Configs.Length);

      for (int i = 0; i < con.ArrayLength; ++i)
      {
        for (int j = 0; j < Collections[i].Lines.Count; ++j)
        {
          var date = Collections[i].Lines[j].Date;
          var val = Collections[i].Lines[j].InvoiceSpread;

          con.SetValue(date, i, val ?? double.PositiveInfinity);
        }
      }

      con.SortKeys();

      // feed forward missing values

      double[] before = null;

      foreach (var date in con.Dates)
      {
        var today = con.GetValues(date);
        if (before != null)
        {
          for(int i=0;i<today.Length;++i)
            if (double.IsInfinity(today[i]))
              today[i] = before[i];
        }
        before = today;
      }

      return con;
    }

    protected override ConstructGen<double> getData(DateTime startDate_, DateTime endDate_, bool forceRefresh_)
    {
      var con = new ConstructGen<double>(Singleton<FXIDs>.Instance.ColumnHeadings);

      var spots = Singleton<FXSpots>.Instance.GetData(startDate_, endDate_, forceRefresh_);
      var fwds = Singleton<FXForwards1Wk>.Instance.GetData(startDate_, endDate_, forceRefresh_);

      var conventions = Singleton<FXIDs>.Instance.ConventionsDs;

      var divisor = 5d;

      foreach (var date in spots.Dates)
      {
        var spotsD = spots.GetValues(date);
        var fwdsD = fwds.GetValues(date);
        var cryD = new double[fwdsD.Length];

        for (int i = 0; i < cryD.Length; ++i)
          if (spotsD[i] != 0d && fwdsD[i] != 0d)
          {
            cryD[i] = (Math.Pow(spotsD[i] / fwdsD[i], conventions[i]) - 1d) * (252d/divisor);
          }

        con.SetValues(date, cryD);
      }

      return con;
    }

    public override void DoWeights(ConstructGen<double> signalResults_, ConstructGen<double> fullSignalResults_, ConstructGen<double> wts_, List<ProductBase> products_, ConstructGen<double> filters_)
    {
      double posSum;
      double negSum;
      int posCount;
      int negCount;

      foreach (DateTime date in wts_.Dates)
      {
        posSum=0.0;
        negSum=0.0;
        posCount=0;
        negCount=0;

        for (int i = 0; i < wts_.ArrayLength; ++i)
        {
          double val = signalResults_.GetValue(date, i);

          if (val == 0)
            continue;

          if (val > 0)
          {
            posSum+=val;
            ++posCount;
          }
          else
          {
            negSum+=val;
            ++negCount;
          }
        }

        double negAv = negSum / negCount;
        double posAv = posSum / posCount;

        for (int i = 0; i < wts_.ArrayLength; ++i)
        {
          double val = signalResults_.GetValue(date, i);

          if (val == 0)
          {
            wts_.MultValue(date, i, 0.0, false);
          }
          else if (val > 0)
          {
            double mult = val / posAv;
            double wt = 1.0 - (1.0 - (1.0 / mult));
            wts_.MultValue(date, i, wt,false);
          }
          else
          {
            double mult = val / negAv;
            double wt = 1.0 - (1.0 - (1.0 / mult));
            wts_.MultValue(date, i, wt,false);
          }
        }
      }
    }

    public void Create(ConstructGen<double> candleData_, int openIndex_ = 0, int highIndex_ = 1, int lowIndex_ = 2, int closeIndex_ = 3, int setupLength_=9, int countdownLength_=13)
    {
      var dt = new DataTable();
      dt.Rows.Clear();
      dt.Columns.Clear();

      dt.Columns.Add("Date", typeof(DateTime));
      dt.Columns.Add("Open", typeof(double));
      dt.Columns.Add("High", typeof(double));
      dt.Columns.Add("Low", typeof(double));
      dt.Columns.Add("Close", typeof(double));
      dt.Columns.Add("Volume", typeof(double));

      ultraChart1.DataSource = dt;

      var closes = candleData_.GetColumnValuesAsDDC(closeIndex_);
      var range = closes.Data.Max() - closes.Data.Min();
      var cellHeight = range/10d;


      var setupStarts = DeMarkAnalysis.GetSetups(candleData_,openIndex_,highIndex_,lowIndex_,closeIndex_,setupLength_);
      DeMarkAnalysis.AddCountdowns(candleData_, setupStarts,openIndex_,highIndex_,lowIndex_,closeIndex_,setupLength_,countdownLength_);


      for (int i = 0; i < candleData_.Dates.Count; ++i)
      {
        var date = candleData_.Dates[i];

        var arr = candleData_.GetValues(date);
        dt.LoadDataRow(new object[]
        {
          date,
          arr[openIndex_],
          arr[highIndex_],
          arr[lowIndex_],
          arr[closeIndex_],
          0d
        }, true);

        foreach(var mark in setupStarts)
        {
          addAnnotations(date, mark, i, cellHeight, arr,openIndex_,highIndex_,lowIndex_,closeIndex_);
        }
      }

      EstablishDefaultTooltip(hash =>
      {
        int rowNumber = (int) hash["DATA_ROW"];

        return string.Format("{0} Open: {1}, High: {2}, Low: {3}, Close: {4}",
          ((DateTime) dt.Rows[rowNumber]["Date"]).ToString("dd-MMM-yyyy"),
          ((double) dt.Rows[rowNumber]["Open"]).ToString(CultureInfo.InvariantCulture),
          ((double)dt.Rows[rowNumber]["High"]).ToString(CultureInfo.InvariantCulture),
          ((double)dt.Rows[rowNumber]["Low"]).ToString(CultureInfo.InvariantCulture),
          ((double)dt.Rows[rowNumber]["Close"]).ToString(CultureInfo.InvariantCulture)
          ).Replace(",", System.Environment.NewLine);

      });
    }

 public void Update(int dateIndex_, ConstructGen<double> data_)
 {
   foreach(TradingLogicNode tcb in m_list)
     if (tcb.Enabled)
     {
       tcb.Update(dateIndex_, data_, m_tradeBasket);
     }
 }

    public void Create(ConstructGen<double> con_)
    {
      setupDataTable();

      double cumulative = 0d;

      Side lastSide = Side.None;

      foreach (var date in con_.Dates)
      {
        var values = con_.GetValues(date);

        var wt = values[(int)Headings.Weight];
        var dailyPnl = values[(int)Headings.Pnl];
        var ccyrate = values[(int)Headings.Spot];

        cumulative += dailyPnl;

        dt.LoadDataRow(new object[] { date, cumulative, ccyrate, wt }, true);

        if (wt > 0d)
        {
          if (lastSide != Side.Long)
          {
            dtLongPoints.LoadDataRow(new object[] { date, ccyrate }, true);
            lastSide = Side.Long;
          }
        }
        else if (wt < 0d)
        {
          if (lastSide != Side.Short)
          {
            dtShortPoints.LoadDataRow(new object[] { date, ccyrate }, true);
            lastSide = Side.Short;
          }
        }
        else
        {
          if (lastSide != Side.Flat)
          {
            dtFlatPoints.LoadDataRow(new object[] { date, ccyrate }, true);
            lastSide = Side.Flat;
          }
        }
      }

      {
        var ccyAxix = ultraChart.CompositeChart.ChartAreas[0].Axes.FromKey("y_axis_ccyRate");

        ccyAxix.RangeType = AxisRangeType.Custom;
        var ccyValues = con_.GetColumnValues((int)Headings.Spot);
        ccyAxix.RangeMin = ccyValues.Min();
        ccyAxix.RangeMax = ccyValues.Max();
      }

      m_con = con_;
    }

    private void loadListFromWeights(ConstructGen<double> wts_, IList<PositionLine<Currency>> list_)
    {
      var lastDate = wts_.Dates.Last();

      var wtsLast = wts_.GetValues(lastDate);

      for (int i = 0; i < wts_.ArrayLength; ++i)
        findIn(list_, Singleton<FXIDs>.Instance[i]).Position = wtsLast[i];
    }