func (self *EruApp) Report() {
	defer self.Client.Close()
	defer logs.Info(self.Name, self.EntryPoint, self.ID[:12], "metrics report stop")
	logs.Info(self.Name, self.EntryPoint, self.ID[:12], "metrics report start")
	for {
		select {
		case now := <-time.Tick(self.Step):
			go func() {
				info, upOk := self.updateStats()
				if isLimit {
					limitChan <- SoftLimit{upOk, self.ID, info}
				}
				if !upOk {
					logs.Info("Update mertic failed", self.Meta.ID[:12])
					return
				}
				rate := self.calcRate(info, now)
				self.saveLast(info)
				// for safe
				go self.send(rate)
			}()
		case <-self.Stop:
			return
		}
	}
}

func (self *EruApp) Report() {
	t := time.NewTicker(self.Step)
	defer t.Stop()
	defer self.Client.Close()
	defer logs.Info(self.Name, self.EntryPoint, self.ID[:12], "metrics report stop")
	logs.Info(self.Name, self.EntryPoint, self.ID[:12], "metrics report start")
	for {
		select {
		case now := <-t.C:
			go func() {
				if info, err := self.UpdateStats(self.ID); err == nil {
					if isLimit {
						limitChan <- SoftLimit{self.ID, info}
					}
					rate := self.CalcRate(info, now)
					self.SaveLast(info)
					go self.Send(rate)
				} else {
					logs.Info("Update mertic failed", self.ID[:12])
				}
			}()
		case <-self.Stop:
			return
		}
	}
}

func SetDefaultRoute(cid, gateway string) bool {
	lock.Lock()
	defer lock.Unlock()
	logs.Info("Set", cid[:12], "default route", gateway)

	container, err := g.Docker.InspectContainer(cid)
	if err != nil {
		logs.Info("RouteSetter inspect docker failed", err)
		return false
	}

	pid := strconv.Itoa(container.State.Pid)
	cmd := exec.Command("nsenter", "-t", pid, "-n", "route", "del", "default")
	if err := cmd.Run(); err != nil {
		logs.Info("Clean default route failed", err)
		return false
	}

	cmd = exec.Command("nsenter", "-t", pid, "-n", "route", "add", "default", "gw", gateway)
	if err := cmd.Run(); err != nil {
		logs.Info("RouteSetter set default route failed", err)
		return false
	}

	logs.Info("Set default route success", cid[:12], gateway)
	return true
}

func getContainerMeta(cid string) map[string]interface{} {
	conn := g.GetRedisConn()
	defer g.ReleaseRedisConn(conn)

	containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName)
	rep, err := gore.NewCommand("HGET", containersKey, cid).Run(conn)
	if err != nil {
		logs.Info("Status get meta", err)
		return nil
	}
	var result map[string]interface{}
	if rep.IsNil() {
		return nil
	}
	if b, err := rep.Bytes(); err != nil {
		logs.Info("Status get meta", err)
		return nil
	} else {
		if err := json.Unmarshal(b, &result); err != nil {
			logs.Info("Status unmarshal meta", err)
			return nil
		}
	}
	return result
}

func monitor() {
	for event := range events {
		switch event.Status {
		case common.STATUS_DIE:
			logs.Debug("Status", event.Status, event.ID[:12], event.From)
			app.Remove(event.ID)
			reportContainerDeath(event.ID)
		case common.STATUS_START:
			logs.Debug("Status", event.Status, event.ID[:12], event.From)
			// if not in watching list, just ignore it
			if meta := getContainerMeta(event.ID); meta != nil && !app.Valid(event.ID) {
				container, err := g.Docker.InspectContainer(event.ID)
				if err != nil {
					logs.Info("Status inspect docker failed", err)
					break
				}
				eruApp := app.NewEruApp(container, meta)
				if eruApp == nil {
					logs.Info("Create EruApp failed")
					break
				}
				lenz.Attacher.Attach(&eruApp.Meta)
				app.Add(eruApp)
				reportContainerCure(event.ID)
			}
		}
	}
}

func routeWatcher() {
	conn := g.GetRedisConn()
	defer g.ReleaseRedisConn(conn)

	subs := gore.NewSubscriptions(conn)
	defer subs.Close()
	subKey := fmt.Sprintf("eru:agent:%s:route", g.Config.HostName)
	logs.Debug("API route subscribe", subKey)
	subs.Subscribe(subKey)

	for message := range subs.Message() {
		if message == nil {
			logs.Info("API route watcher shutdown")
			break
		}
		command := string(message.Message)
		logs.Debug("API route watcher get", command)
		parser := strings.Split(command, "|")
		if len(parser) != 2 {
			logs.Info("API route watcher command invaild", command)
			continue
		}
		cid, gateway := parser[0], parser[1]
		if !network.SetDefaultRoute(cid, gateway) {
			logs.Info("Set default route failed")
		}
	}
}

func (self *EruApp) updateStats() bool {
	statsChan := make(chan *docker.Stats)
	opt := docker.StatsOptions{self.ID, statsChan, false, nil, time.Duration(2 * time.Second)}
	go func() {
		if err := g.Docker.Stats(opt); err != nil {
			logs.Info("Get Stats Failed", err)
		}
	}()
	stats := <-statsChan
	if stats == nil {
		return false
	}

	self.Info["cpu_user"] = stats.CPUStats.CPUUsage.UsageInUsermode
	self.Info["cpu_system"] = stats.CPUStats.CPUUsage.UsageInKernelmode
	self.Info["cpu_usage"] = stats.CPUStats.CPUUsage.TotalUsage
	//FIXME in container it will get all CPUStats
	//	for seq, d := range stats.CPUStats.CPUUsage.PercpuUsage {
	//		self.Info[fmt.Sprintf("cpu_%d", seq)] = d
	//	}
	self.Info["mem_usage"] = stats.MemoryStats.Usage
	self.Info["mem_max_usage"] = stats.MemoryStats.MaxUsage
	self.Info["mem_rss"] = stats.MemoryStats.Stats.Rss

	network, err := GetNetStats(self.Exec)
	if err != nil {
		logs.Info(err)
		return false
	}
	for k, d := range network {
		self.Info[k] = d
	}
	return true
}

// URL /api/container/add/
func addNewContainer(req *Request) (int, interface{}) {
	type Data struct {
		Control     string                 `json:"control"`
		ContainerID string                 `json:"container_id"`
		Meta        map[string]interface{} `json:"meta"`
	}

	data := &Data{}
	decoder := json.NewDecoder(req.Body)
	err := decoder.Decode(data)
	if err != nil {
		return http.StatusBadRequest, JSON{"message": "wrong JSON format"}
	}

	switch data.Control {
	case "+":
		if app.Valid(data.ContainerID) {
			break
		}
		logs.Info("API status watch", data.ContainerID)
		container, err := g.Docker.InspectContainer(data.ContainerID)
		if err != nil {
			logs.Info("API status inspect docker failed", err)
			break
		}
		if eruApp := app.NewEruApp(container.ID, container.Name, data.Meta); eruApp != nil {
			app.Add(eruApp)
			lenz.Attacher.Attach(&eruApp.Meta)
		}
	}
	return http.StatusOK, JSON{"message": "ok"}
}

func AddRoute(cid, CIDR string, ifc string) bool {
	_, err := g.Docker.InspectContainer(cid)
	if err != nil {
		logs.Info("VLanSetter inspect docker failed", err)
		return false
	}
	logs.Info("Add route success", cid, CIDR, ifc)
	return true
}

func AddVLan(vethName, ips, cid string) bool {
	_, err := g.Docker.InspectContainer(cid)
	if err != nil {
		logs.Info("VLanSetter inspect docker failed", err)
		return false
	}
	logs.Info("Add VLAN device success", cid, vethName)
	return true
}

func SetDefaultRoute(cid, gateway string) bool {
	_, err := g.Docker.InspectContainer(cid)
	if err != nil {
		logs.Info("VLanSetter inspect docker failed", err)
		return false
	}
	logs.Info("Set default route success", cid, gateway)
	return true
}

func load() {
	containers, err := g.Docker.ListContainers(docker.ListContainersOptions{All: true})
	if err != nil {
		logs.Assert(err, "List containers")
	}

	conn := g.GetRedisConn()
	defer g.ReleaseRedisConn(conn)
	containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName)
	logs.Debug("Status get targets from", containersKey)
	rep, err := gore.NewCommand("HGETALL", containersKey).Run(conn)
	if err != nil {
		logs.Assert(err, "Status get targets")
	}

	if rep.IsNil() {
		return
	}

	targets, err := rep.Map()
	if err != nil {
		logs.Assert(err, "Status load targets")
	}

	logs.Debug("Status targets:", targets)
	logs.Info("Status load container")
	for _, container := range containers {
		if _, ok := targets[container.ID]; !ok {
			continue
		}

		status := getStatus(container.Status)
		if status != common.STATUS_START {
			reportContainerDeath(container.ID)
			continue
		}
		var meta map[string]interface{}
		if err := json.Unmarshal([]byte(targets[container.ID]), &meta); err != nil {
			logs.Info("Status load failed", err)
			continue
		}

		c, err := g.Docker.InspectContainer(container.ID)
		if err != nil {
			logs.Info("Status inspect docker failed", err)
			continue
		}

		if eruApp := app.NewEruApp(c, meta); eruApp != nil {
			lenz.Attacher.Attach(&eruApp.Meta)
			app.Add(eruApp)
			reportContainerCure(container.ID)
		}
	}
}

func Streamer(route *defines.Route, logstream chan *defines.Log, stdout bool) {
	var types map[string]struct{}
	var count int64 = 0
	if route.Source != nil {
		types = make(map[string]struct{})
		for _, t := range route.Source.Types {
			types[t] = struct{}{}
		}
	}
	for logline := range logstream {
		if types != nil {
			if _, ok := types[logline.Type]; !ok {
				continue
			}
		}
		logline.Tag = route.Target.AppendTag
		logline.Count = count

		switch stdout {
		case true:
			logs.Info("Debug Output", logline)
		default:
			for offset := 0; offset < route.Backends.Len(); offset++ {
				addr, err := route.Backends.Get(logline.Name, offset)
				if err != nil {
					logs.Info("Get backend failed", err, logline.Name, logline.Data)
					break
				}
				if _, ok := upstreams[addr]; !ok {
					if ups, err := NewUpStream(addr); err != nil || ups == nil {
						route.Backends.Remove(addr)
						continue
					} else {
						upstreams[addr] = ups
					}
				}
				if err := upstreams[addr].WriteData(logline); err != nil {
					upstreams[addr].Close()
					delete(upstreams, addr)
					continue
				}
				//logs.Debug("Lenz Send", logline.Name, logline.EntryPoint, logline.ID, "to", addr)
				break
			}
		}
		if count == math.MaxInt64 {
			count = 0
		} else {
			count++
		}
	}
}

func CloseLenz() {
	logs.Info("Close all lenz streamer")
	routes, err := Router.GetAll()
	if err != nil {
		logs.Info("Get all lenz route failed", err)
		return
	}
	for _, route := range routes {
		if !Router.Remove(route.ID) {
			logs.Info("Close lenz route failed", route.ID)
		}
	}
}

func softOOMKill(cid string, rate float64) {
	logs.Debug("OOM killed", cid[:12])
	conn := g.GetRedisConn()
	defer g.ReleaseRedisConn(conn)

	key := fmt.Sprintf("eru:agent:%s:container:reason", cid)
	if _, err := gore.NewCommand("SET", key, common.OOM_KILLED).Run(conn); err != nil {
		logs.Info("OOM killed set flag", err)
	}
	if err := g.Docker.StopContainer(cid, 10); err != nil {
		logs.Info("OOM killed failed", cid[:12])
		return
	}
	logs.Info("OOM killed success", cid[:12])
}

func (self *EruApp) InitMetric() bool {
	var err error
	if self.statFile, err = os.Open(fmt.Sprintf("/proc/%d/net/dev", self.Meta.Pid)); err != nil {
		logs.Info("Open net stats failed", self.Meta.ID[:12])
		return false
	}
	info, upOk := self.updateStats()
	if !upOk {
		logs.Info("Init mertics failed", self.Meta.ID[:12])
		return false
	}
	self.Last = time.Now()
	self.saveLast(info)
	return true
}

func InitTransfers() {
	Transfers = consistent.New()
	for _, transfer := range Config.Metrics.Transfers {
		Transfers.Add(transfer)
	}
	logs.Info("Metrics initiated")
}

func HTTPServe() {
	restfulAPIServer := pat.New()

	handlers := map[string]map[string]func(*Request) (int, interface{}){
		"GET": {
			"/profile/":      profile,
			"/version/":      version,
			"/api/app/list/": listEruApps,
		},
		"POST": {
			"/api/container/add/":                    addNewContainer,
			"/api/container/:container_id/addvlan/":  addVlanForContainer,
			"/api/container/:container_id/setroute/": setRouteForContainer,
		},
	}

	for method, routes := range handlers {
		for route, handler := range routes {
			restfulAPIServer.Add(method, route, http.HandlerFunc(JSONWrapper(handler)))
		}
	}

	http.Handle("/", restfulAPIServer)
	logs.Info("API http server start at", g.Config.API.Addr)
	err := http.ListenAndServe(g.Config.API.Addr, nil)
	if err != nil {
		logs.Assert(err, "ListenAndServe: ")
	}
}

func (self *Metric) UpdateStats(cid string) (map[string]uint64, error) {
	info := map[string]uint64{}
	statsChan := make(chan *docker.Stats)
	doneChan := make(chan bool)
	opt := docker.StatsOptions{cid, statsChan, false, doneChan, g.timeout * time.Second}
	go func() {
		if err := g.client.Stats(opt); err != nil {
			logs.Info("Get stats failed", cid[:12], err)
		}
	}()

	var stats *docker.Stats = nil
	select {
	case stats = <-statsChan:
		if stats == nil {
			return info, errors.New("Get stats failed")
		}
	case <-time.After(g.force * time.Second):
		doneChan <- true
		return info, errors.New("Get stats timeout")
	}

	info["cpu_user"] = stats.CPUStats.CPUUsage.UsageInUsermode
	info["cpu_system"] = stats.CPUStats.CPUUsage.UsageInKernelmode
	info["cpu_usage"] = stats.CPUStats.CPUUsage.TotalUsage
	//FIXME in container it will get all CPUStats
	info["mem_usage"] = stats.MemoryStats.Usage
	info["mem_max_usage"] = stats.MemoryStats.MaxUsage
	info["mem_rss"] = stats.MemoryStats.Stats.Rss

	if err := self.getNetStats(info); err != nil {
		return info, err
	}
	return info, nil
}

func Limit() {
	if g.Config.Limit.Memory != 0 {
		logs.Info("App memory soft limit start")
		isLimit = true
		go calcMemoryUsage()
	}
}