func (t *Tsdb) Flush() {
	t.Lock()
	if len(t.Metrics) == 0 {
		t.Unlock()
		return
	}
	metrics := make([]*schema.MetricData, len(t.Metrics))
	copy(metrics, t.Metrics)
	t.Metrics = t.Metrics[:0]
	t.Unlock()

	// Write the metrics to our HTTP server.
	log.Debug("writing %d metrics to API", len(metrics))
	batches := schema.Reslice(metrics, maxMetricsPerFlush*2)
	for _, batch := range batches {
		id := time.Now().UnixNano()
		body, err := msg.CreateMsg(batch, id, msg.FormatMetricDataArrayMsgp)
		if err != nil {
			log.Error(3, "unable to convert metrics to MetricDataArrayMsgp.", "error", err)
			return
		}
		t.dataChan <- tsdbData{Path: "metrics", Body: body}
		log.Debug("%d metrics queud for delivery", len(batch))
	}
}

func socket(ctx *Context) {
	agentName := ctx.Params(":agent")
	agentVer := ctx.ParamsInt64(":ver")
	//TODO: add auth
	owner := ctx.OrgId
	agent, err := connectedAgent(agentName, owner)
	if err != nil {
		log.Debug("agent cant connect. %s", err)
		ctx.JSON(400, err.Error())
		return
	}

	c, err := upgrader.Upgrade(ctx.Resp, ctx.Req.Request, nil)
	if err != nil {
		log.Error(3, "upgrade:", err)
		return
	}

	log.Debug("agent %s connected.", agent.Name)

	sess := agent_session.NewSession(agent, agentVer, c)
	ActiveSockets.NewSocket(sess)
	sess.Start()
	//block until connection closes.
	<-sess.Done
	ActiveSockets.DeleteSocket(sess)
}

func (t *Tsdb) Run() {
	for i := 0; i < t.Concurrency; i++ {
		go t.sendData()
	}

	ticker := time.NewTicker(time.Second)
	last := time.Now()
	for {
		select {
		case <-ticker.C:
			if time.Since(last) >= time.Second {
				log.Debug("no flushes in last 1second. Flushing now.")
				last = time.Now()
				t.Flush()
				log.Debug("flush took %f seconds", time.Since(last).Seconds())
			}
		case <-t.flushMetrics:
			log.Debug("flush trigger received.")
			last = time.Now()
			t.Flush()
			log.Debug("flush took %f seconds", time.Since(last).Seconds())
		case <-t.flushEvents:
			t.SendEvents()
		case <-t.closeChan:
			close(t.dataChan)
			return
		}
	}
}

func (s *socketList) EmitTask(task *model.TaskDTO, event string) error {
	log.Debug("sending %s task event to connected agents.", event)
	agents, err := sqlstore.GetAgentsForTask(task)
	log.Debug("Task has %d agents. %v", len(agents), agents)
	if err != nil {
		return err
	}
	body, err := json.Marshal(task)
	if err != nil {
		return err
	}
	e := &message.Event{
		Event:   event,
		Payload: body,
	}
	sent := false
	s.Lock()

	for _, id := range agents {
		if as, ok := s.Sockets[id]; ok {
			log.Debug("sending %s event to agent %d", event, id)
			as.SocketSession.Emit(e)
			sent = true
		} else {
			log.Debug("agent %d is not connected to this server.", id)
		}
	}
	s.Unlock()
	if !sent {
		log.Debug("no connected agents for task %d.", task.Id)
	}
	return nil
}

func (t *TaskCache) addTask(task *model.TaskDTO) error {
	t.Tasks[task.Id] = task
	if !t.initialized {
		return nil
	}
	snapTaskName := fmt.Sprintf("raintank-apps:%d", task.Id)
	snapTask, ok := t.SnapTasks[snapTaskName]
	if !ok {
		log.Debug("New task recieved %s", snapTaskName)
		snapTask, err := t.c.CreateSnapTask(task, snapTaskName)
		if err != nil {
			return err
		}
		t.SnapTasks[snapTaskName] = snapTask
	} else {
		log.Debug("task %s already in the cache.", snapTaskName)
		if task.Updated.After(time.Unix(snapTask.CreationTimestamp, 0)) {
			log.Debug("%s needs to be updated", snapTaskName)
			// need to update task.
			if err := t.c.RemoveSnapTask(snapTask); err != nil {
				return err
			}
			snapTask, err := t.c.CreateSnapTask(task, snapTaskName)
			if err != nil {
				return err
			}
			t.SnapTasks[snapTaskName] = snapTask
		}
	}

	return nil
}

func (a *AgentSession) Start() error {
	if err := a.saveDbSession(); err != nil {
		log.Error(3, "unable to add agentSession to DB. %s", err.Error())
		a.close()
		return err
	}

	log.Debug("setting handler for disconnect event.")
	if err := a.SocketSession.On("disconnect", a.OnDisconnect()); err != nil {
		log.Error(3, "failed to bind disconnect event. %s", err.Error())
		a.close()
		return err
	}

	log.Debug("setting handler for catalog event.")
	if err := a.SocketSession.On("catalog", a.HandleCatalog()); err != nil {
		log.Error(3, "failed to bind catalog event handler. %s", err.Error())
		a.close()
		return err
	}

	log.Info("starting session %s", a.SocketSession.Id)
	go a.SocketSession.Start()

	// run background tasks for this session.
	go a.sendHeartbeat()
	go a.sendTaskListPeriodically()
	a.sendTaskList()
	return nil
}

// redial continually connects to the URL, exiting the program when no longer possible
func redial(ctx context.Context, url, exchange string) chan chan session {
	sessions := make(chan chan session)

	go func() {
		sess := make(chan session)
		defer close(sessions)

		for {
			select {
			case sessions <- sess:
			case <-ctx.Done():
				log.Info("shutting down session factory")
				return
			}

			connected := false
			var conn *amqp.Connection
			var ch *amqp.Channel
			var err error
			for !connected {
				log.Debug("dialing amqp url: %s", url)
				conn, err = amqp.Dial(url)
				if err != nil {
					log.Error(3, "cannot (re)dial: %v: %q", err, url)
					time.Sleep(time.Second)
					continue
				}
				log.Debug("connected to %s", url)

				log.Debug("creating new channel on AMQP connection.")
				ch, err = conn.Channel()
				if err != nil {
					log.Error(3, "cannot create channel: %v", err)
					conn.Close()
					time.Sleep(time.Second)
					continue
				}
				log.Debug("Ensuring that %s topic exchange exists on AMQP server.", exchange)
				if err := ch.ExchangeDeclare(exchange, "topic", true, false, false, false, nil); err != nil {
					log.Error(3, "cannot declare topic exchange: %v", err)
					conn.Close()
					time.Sleep(time.Second)
				}
				log.Debug("Successfully connected to RabbitMQ.")
				connected = true
			}

			select {
			case sess <- session{conn, ch}:
			case <-ctx.Done():
				log.Info("shutting down new session")
				return
			}
		}
	}()

	return sessions
}

func (a *AgentSession) cleanup() {
	//remove agentSession from DB.
	if a.dbSession != nil {
		log.Debug("deleting agent_session for %s from DB", a.Agent.Name)
		sqlstore.DeleteAgentSession(a.dbSession)
	} else {
		log.Debug("agent_session for %s has no db session.", a.Agent.Name)
	}
}

func (s *socketList) NewSocket(a *agent_session.AgentSession) {
	s.Lock()
	existing, ok := s.Sockets[a.Agent.Id]
	if ok {
		log.Debug("new connection for agent %d - %s, closing existing session", a.Agent.Id, a.Agent.Name)
		existing.Close()
	}
	log.Debug("Agent %d is connected to this server.", a.Agent.Id)
	s.Sockets[a.Agent.Id] = a
	s.Unlock()
}

func (a *AgentSession) close() {
	if !a.closing {
		a.closing = true
		close(a.Shutdown)
		log.Debug("closing websocket")
		a.SocketSession.Close()
		log.Debug("websocket closed")

		a.cleanup()
		close(a.Done)
	}
}

func Publish(metrics []*schema.MetricData) error {
	if globalProducer == nil {
		log.Debug("droping %d metrics as publishing is disbaled", len(metrics))
		return nil
	}
	if len(metrics) == 0 {
		return nil
	}

	subslices := schema.Reslice(metrics, 3500)

	for _, subslice := range subslices {
		id := time.Now().UnixNano()
		data, err := msg.CreateMsg(subslice, id, msg.FormatMetricDataArrayMsgp)
		if err != nil {
			log.Fatal(4, "Fatal error creating metric message: %s", err)
		}
		metricsPublished.Inc(int64(len(subslice)))
		messagesPublished.Inc(1)
		messagesSize.Value(int64(len(data)))
		metricsPerMessage.Value(int64(len(subslice)))
		pre := time.Now()
		err = globalProducer.Publish(topic, data)
		publishDuration.Value(time.Since(pre))
		if err != nil {
			log.Fatal(4, "can't publish to nsqd: %s", err)
		}
		log.Info("published metrics %d size=%d", id, len(data))
	}

	//globalProducer.Stop()
	return nil
}

func (t *Tsdb) Add(metrics []*schema.MetricData) {
	log.Debug("received %d new metrics", len(metrics))
	t.Lock()
	t.Metrics = append(t.Metrics, metrics...)
	numMetrics := len(t.Metrics)
	t.Unlock()
	if numMetrics > maxMetricsPerFlush {
		//non-blocking send on the channel. If there is already
		// an item in the channel we dont need to add another.
		select {
		default:
			log.Debug("flushMetrics channel blocked.")
		case t.flushMetrics <- struct{}{}:
		}
	}
}

func (s *socketList) CloseSocketByAgentId(id int64) {
	s.Lock()
	existing, ok := s.Sockets[id]
	if ok {
		existing.Close()
		log.Debug("removing session for Agent %d from socketList.", id)
		delete(s.Sockets, id)
	}
	s.Unlock()
}

func (s *socketList) CloseSocket(a *agent_session.AgentSession) {
	s.Lock()
	existing, ok := s.Sockets[a.Agent.Id]
	if ok {
		existing.Close()
		log.Debug("removing session for Agent %d from socketList.", a.Agent.Id)
		delete(s.Sockets, a.Agent.Id)
	}
	s.Unlock()
}

// publish publishes messages to a reconnecting session to a topic exchange.
// It receives from the application specific source of messages.
func publish(sessions chan chan session, exchange string, messages <-chan Message) {
	var (
		running bool
		reading = messages
		pending = make(chan Message, 1)
		confirm = make(chan amqp.Confirmation, 1)
	)

	for session := range sessions {
		log.Debug("waiting for new session to be established.")
		pub := <-session

		// publisher confirms for this channel/connection
		if err := pub.Confirm(false); err != nil {
			log.Info("publisher confirms not supported")
			close(confirm) // confirms not supported, simulate by always nacking
		} else {
			pub.NotifyPublish(confirm)
		}

		log.Info("Event publisher started...")

		for {
			var body Message
			select {
			case confirmed := <-confirm:
				if !confirmed.Ack {
					log.Error(3, "nack message %d, body: %q", confirmed.DeliveryTag, string(body.Payload))
				}
				reading = messages

			case body = <-pending:
				err := pub.Publish(exchange, body.RoutingKey, false, false, amqp.Publishing{
					Body: body.Payload,
				})
				// Retry failed delivery on the next session
				if err != nil {
					pending <- body
					pub.Close()
					break
				}

			case body, running = <-reading:
				// all messages consumed
				if !running {
					return
				}
				// work on pending delivery until ack'd
				pending <- body
				reading = nil
			}
		}
	}
}

func (s *Scheduler) Refresh(checks []*m.CheckWithSlug) {
	log.Info("refreshing checks, there are %d", len(checks))
	seenChecks := make(map[int64]struct{})
	s.Lock()
	for _, c := range checks {
		if !c.Enabled {
			continue
		}
		seenChecks[c.Id] = struct{}{}
		if existing, ok := s.Checks[c.Id]; ok {
			log.Debug("checkId=%d already running", c.Id)
			if c.Updated.After(existing.Check.Updated) {
				log.Info("syncing update to checkId=%d", c.Id)
				err := existing.Update(c, s.Healthy)
				if err != nil {
					log.Error(3, "Unable to update check instance for checkId=%d", c.Id, err)
					existing.Stop()
					delete(s.Checks, c.Id)
				}
			}
		} else {
			log.Debug("new check definition found for checkId=%d.", c.Id)
			instance, err := NewCheckInstance(c, s.Healthy)
			if err != nil {
				log.Error(3, "Unabled to create new check instance for checkId=%d.", c.Id, err)
			} else {
				s.Checks[c.Id] = instance
			}
		}
	}
	for id, instance := range s.Checks {
		if _, ok := seenChecks[id]; !ok {
			log.Info("checkId=%d no longer scheduled to this probe, removing it.", id)
			instance.Stop()
			delete(s.Checks, id)
		}
	}
	s.Unlock()
	log.Debug("refresh complete")
	return
}

func HandleTaskList() interface{} {
	return func(data []byte) {
		tasks := make([]*model.TaskDTO, 0)
		err := json.Unmarshal(data, &tasks)
		if err != nil {
			log.Error(3, "failed to decode taskUpdate payload. %s", err)
			return
		}
		log.Debug("TaskList. %s", data)
		GlobalTaskCache.UpdateTasks(tasks)
	}
}

func (t *TaskCache) RemoveTask(task *model.TaskDTO) error {
	t.Lock()
	defer t.Unlock()
	snapTaskName := fmt.Sprintf("raintank-apps:%d", task.Id)
	log.Debug("removing snap task %s", snapTaskName)
	if err := t.removeSnapTask(snapTaskName); err != nil {
		return err
	}

	delete(t.Tasks, task.Id)
	return nil
}

func (t *Tsdb) AddEvent(event *schema.ProbeEvent) {
	t.Lock()
	t.Events = append(t.Events, event)
	t.Unlock()
	//non-blocking send on the channel. If there is already
	// an item in the channel we dont need to add another.
	select {
	default:
		log.Debug("flushEvents channel blocked.")
	case t.flushEvents <- struct{}{}:
	}
}

func (a *AgentSession) sendTaskListPeriodically() {
	ticker := time.NewTicker(time.Second * 60)
	for {
		select {
		case <-a.Shutdown:
			log.Debug("session ended stopping taskListPeriodically.")
			return
		case <-ticker.C:
			a.sendTaskList()
		}
	}
}