func (sc *ServerCodec) ReadRequestBody(v interface{}) (err error) {
	log.Println(log.TRACE, "RPC Server Entered: ReadRequestBody")
	defer log.Println(log.TRACE, "RPC Server Leaving: ReadRequestBody")

	err = sc.Decoder.Decode(v)
	if err != nil {
		log.Println(log.ERROR, "RPC Server Error decoding request body: ", err)
	}

	if err == nil {
		log.Println(log.TRACE, pretty.Sprintf("RPC Server Read RequestBody %s %+v", reflect.TypeOf(v), v))
	}
	return
}

func (s *SkynetDaemon) StopAllSubServices(requestInfo *skynet.RequestInfo, in daemon.StopAllSubServicesRequest, out *daemon.StopAllSubServicesResponse) (err error) {
	var uuids []string
	s.serviceLock.Lock()
	for uuid := range s.Services {
		uuids = append(uuids, uuid)
	}
	s.serviceLock.Unlock()

	out.Stops = make([]daemon.StopSubServiceResponse, len(uuids))

	for i, uuid := range uuids {
		log.Println(log.TRACE, "Stopping "+uuid)
		err = s.StopSubService(requestInfo, daemon.StopSubServiceRequest{UUID: uuid}, &out.Stops[i])
		if err != nil {
			log.Println(log.ERROR, "Failed to stop subservice "+uuid, err)
			return
		}
		if out.Stops[i].Ok {
			out.Count++
		}
	}

	s.saveState()

	return
}

// TODO: This should be moved out so that it's run asynchronously
// it should also use a buffered channel so that if a save is already queued it only saves once
func (s *SkynetDaemon) writeStateFile() (err error) {
	err = s.stateFile.Truncate(0)

	if err != nil {
		return
	}

	_, err = s.stateFile.Seek(0, 0)

	if err != nil {
		return
	}

	var b []byte
	b, err = json.MarshalIndent(s.Services, "", "\t")

	if err != nil {
		log.Println(log.ERROR, "Failed to marshall daemon state")
		return
	}

	_, err = s.stateFile.Write(b)

	if err != nil {
		log.Println(log.ERROR, "Failed to save daemon state")
	}

	return
}

func (c *ServiceClient) send(retry, giveup time.Duration, ri *skynet.RequestInfo, fn string, in interface{}, out interface{}) (err error) {
	if ri == nil {
		ri = c.NewRequestInfo()
	}

	attempts := make(chan sendAttempt)

	var retryTicker <-chan time.Time
	retryChan := make(chan bool, 1)
	if retry > 0 {
		retryTicker = time.Tick(retry)
	}

	var timeoutTimer <-chan time.Time
	if giveup > 0 {
		timeoutTimer = time.NewTimer(giveup).C
	}

	attemptCount := 1
	go c.attemptSend(retry, attempts, ri, fn, in, out)

	for {
		select {
		case <-retryTicker:
			retryChan <- true

		case <-retryChan:
			attemptCount++
			ri.RetryCount++
			log.Println(log.TRACE, fmt.Sprintf("Sending Attempt# %d with RequestInfo %+v", attemptCount, ri))
			go c.attemptSend(retry, attempts, ri, fn, in, out)

		case <-timeoutTimer:
			err = RequestTimeout
			log.Println(log.WARN, fmt.Sprintf("Timing out request after %d attempts within %s ", attemptCount, giveup.String()))
			return

		case attempt := <-attempts:
			if attempt.err != nil {
				log.Println(log.ERROR, "Attempt Error: ", attempt.err)

				// If there is no retry timer we need to exit as retries were disabled
				if retryTicker == nil {
					return err
				} else {
					// Don't wait for next retry tick retry now
					retryChan <- true
				}

				continue
			}

			// copy into the caller's value
			v := reflect.Indirect(reflect.ValueOf(out))
			v.Set(reflect.Indirect(reflect.ValueOf(attempt.result)))

			return
		}
	}
}

func (pc *PathCache) Start() (notifyChan chan PathCacheNotification, err error) {
	pc.notifyChan = make(chan PathCacheNotification, 10)
	pc.stoppedChan = make(chan bool, 1)
	pc.startup = sync.WaitGroup{}

	go pc.mux()

	err = pc.watch()

	if err != nil {
		// We expect this to occasionally happen due to timing
		if err != zk.ErrNoNode {
			log.Println(log.ERROR, err)
		}

		pc.Stop()
		return
	}

	err = pc.watchChildren()

	if err != nil {
		// We expect this to occasionally happen due to timing
		if err != zk.ErrNoNode {
			log.Println(log.ERROR, err)
		}
		pc.Stop()
	}

	pc.startup.Wait()

	return pc.notifyChan, err
}

func init() {
	flagset := flag.NewFlagSet("config", flag.ContinueOnError)
	flagset.StringVar(&configFile, "config", "", "Config File")
	flagset.StringVar(&uuid, "uuid", "", "uuid")

	args, _ := SplitFlagsetFromArgs(flagset, os.Args[1:])
	flagset.Parse(args)

	// Ensure we have a UUID
	if uuid == "" {
		uuid = NewUUID()
	}

	if configFile == "" {
		for _, f := range defaultConfigFiles {
			if _, err := os.Stat(f); err == nil {
				configFile = f
				break
			}
		}
	}

	if configFile == "" {
		log.Println(log.ERROR, "Failed to find config file")
		conf = config.NewDefault()
		return
	}

	if _, err := os.Stat(configFile); os.IsNotExist(err) {
		log.Println(log.ERROR, "Config file does not exist", err)
		conf = config.NewDefault()
		return
	}

	var err error
	if conf, err = config.ReadDefault(configFile); err != nil {
		conf = config.NewDefault()
		log.Fatal(err)
	}

	// Set default log level from config, this can be overriden at the service level when the service is created
	if l, err := conf.RawStringDefault("log.level"); err == nil {
		log.SetLogLevel(log.LevelFromString(l))
	}

	// Set default log level from config, this can be overriden at the service level when the service is created
	if lh, err := conf.RawStringDefault("log.sysloghost"); err == nil {
		log.SetSyslogHost(lh)
	}

	// Set syslog port
	if i, err := conf.Int("DEFAULT", "log.syslogport"); err == nil {
		log.SetSyslogPort(i)
	}

	// Set GOMAXPROCS
	if i, err := conf.Int("DEFAULT", "runtime.gomaxprocs"); err == nil {
		runtime.GOMAXPROCS(i)
	}
}

func (sc *ServerCodec) Close() (err error) {
	log.Println(log.TRACE, "RPC Server Entered: Close")
	defer log.Println(log.TRACE, "RPC Server Leaving: Close")

	err = sc.conn.Close()
	if err != nil && err.Error() != "use of closed network connection" {
		log.Println(log.ERROR, "RPC Server Error closing connection: ", err)
		return
	}
	return
}

func getGiveupTimeout(service, version string) time.Duration {
	if d, err := config.String(service, version, "client.timeout.total"); err == nil {
		if timeout, err := time.ParseDuration(d); err == nil {
			log.Println(log.TRACE, fmt.Sprintf("Using custom giveup duration %q for %q %q", timeout.String(), service, version))
			return timeout
		}

		log.Println(log.ERROR, "Failed to parse client.timeout.total", err)
	}

	return config.DefaultTimeoutDuration
}

func (cc *ClientCodec) Close() (err error) {
	log.Println(log.TRACE, "RPC Client Entered: Close")
	defer log.Println(log.TRACE, "RPC Client Leaving: Close")

	err = cc.conn.Close()

	if err != nil && err.Error() != "use of closed network connection" {
		log.Println(log.ERROR, "RPC Client Error closing connection: ", err)
	}

	return
}

func (sd *SkynetDaemon) Started(s *service.Service) {
	err := sd.cleanupHost(s.ServiceInfo.UUID)
	if err != nil {
		log.Println(log.ERROR, "Error cleaning up host", err)
	}

	err = sd.restoreState()

	if err != nil {
		log.Println(log.ERROR, "Error restoring state", err)
	}
}

func (sc *ServerCodec) ReadRequestHeader(rq *rpc.Request) (err error) {
	log.Println(log.TRACE, "RPC Server Entered: ReadRequestHeader")
	defer log.Println(log.TRACE, "RPC Server Leaving: ReadRequestHeader")

	err = sc.Decoder.Decode(rq)
	if err != nil && err != io.EOF {
		log.Println(log.ERROR, "RPC Server Error decoding request header: ", err)
		sc.Close()
	}

	if err == nil {
		log.Println(log.TRACE, pretty.Sprintf("RPC Server Read RequestHeader %s %+v", reflect.TypeOf(rq), rq))
	}
	return
}

func (pc *PathCache) watchChildren() error {
	if pc.depth == 0 {
		return nil
	}

	children, _, ev, err := pc.serviceManager.conn.ChildrenW(pc.path)

	if err != nil {
		if err != zk.ErrNoNode {
			log.Println(log.ERROR, err)
		}
		return err
	}

	go forwardZkEvents(ev, pc.events)

	pc.startup.Add(len(children))
	for _, c := range children {
		if _, ok := pc.children[path.Join(pc.path, c)]; !ok {
			go func(c string) {
				pc.addChildChan <- c
			}(c)
		}
	}

	return nil
}

func (sm *ZookeeperServiceManager) Unregister(uuid string) (err error) {
	log.Println(log.TRACE, "Unregister service", uuid)

	_, err = sm.conn.Set(path.Join("/instances", uuid, "registered"), []byte("false"), -1)

	return
}

func (sm *ZookeeperServiceManager) Remove(s skynet.ServiceInfo) (err error) {
	log.Println(log.TRACE, "Removing service", s.UUID)

	ops := zk.MultiOps{
		Delete: []zk.DeleteRequest{
			deleteRequest(path.Join("/regions", s.Region, s.UUID), -1),
			deleteRequest(path.Join("/services", s.Name, s.Version, s.UUID), -1),
			deleteRequest(path.Join("/services", s.Name, s.UUID), -1),
			deleteRequest(path.Join("/hosts", s.ServiceAddr.IPAddress, s.UUID), -1),

			deleteRequest(path.Join("/instances", s.UUID, "registered"), -1),
			deleteRequest(path.Join("/instances", s.UUID, "name"), -1),
			deleteRequest(path.Join("/instances", s.UUID, "version"), -1),
			deleteRequest(path.Join("/instances", s.UUID, "region"), -1),
			deleteRequest(path.Join("/instances", s.UUID, "addr"), -1),

			deleteRequest(path.Join("/instances", s.UUID), -1),
		},
	}

	err = sm.conn.Multi(ops)

	if err == nil {
		delete(sm.managedInstances, s.UUID)
	}

	// Attempt to remove parent paths for service if they are empty
	sm.removePathIfEmpty(path.Join("/hosts", s.ServiceAddr.IPAddress))
	sm.removePathIfEmpty(path.Join("/regions", s.Region))
	sm.removePathIfEmpty(path.Join("/services", s.Name, s.Version))
	sm.removePathIfEmpty(path.Join("/services", s.Name))

	return
}

func (sm *ZookeeperServiceManager) mux() {
	for {
		select {
		case e := <-sm.session:
			switch e.Type {
			case zk.EventNodeDeleted, zk.EventNodeChildrenChanged, zk.EventNodeDataChanged:
			case zk.EventSession:
			// TODO: EventNotWatching
			// TODO: StateDisconnected
			default:
				log.Println(log.TRACE, "Zookeeper Event Received: ", e)
			}
		case <-sm.done:
			// Remove instances that were added by this instance
			for _, s := range sm.managedInstances {
				sm.Remove(s)
			}

			sm.cache.Stop()

			sm.conn.Close()
			return
		}
	}
}

// Wait for existing requests to complete and shutdown service
func (s *Service) shutdown() {
	if s.shuttingDown {
		return
	}

	s.shuttingDown = true

	s.doneGroup.Add(1)
	s.rpcListener.Close()

	s.doneChan <- true

	s.activeRequests.Wait()

	err := skynet.GetServiceManager().Remove(*s.ServiceInfo)
	if err != nil {
		log.Println(log.ERROR, "Failed to remove service: "+err.Error())
	}

	skynet.GetServiceManager().Shutdown()

	s.Delegate.Stopped(s) // Call user defined callback

	s.doneGroup.Done()
}

func (s *Service) listen(addr skynet.BindAddr, bindWait *sync.WaitGroup) {
	var err error
	s.rpcListener, err = addr.Listen()
	if err != nil {
		log.Fatal(err)
	}

	log.Printf(log.INFO, "%+v\n", ServiceListening{
		Addr:        &addr,
		ServiceInfo: s.ServiceInfo,
	})

	// We may have changed port due to conflict, ensure config has the correct port now
	a, _ := skynet.BindAddrFromString(addr.String())
	s.ServiceAddr.IPAddress = a.IPAddress
	s.ServiceAddr.Port = a.Port

	bindWait.Done()

	for {
		conn, err := s.rpcListener.AcceptTCP()

		if s.shuttingDown {
			break
		}

		if err != nil && !s.shuttingDown {
			log.Println(log.ERROR, "AcceptTCP failed", err)
			continue
		}
		s.connectionChan <- conn
	}
}

func (sm *ZookeeperServiceManager) Add(s skynet.ServiceInfo) (err error) {
	log.Println(log.TRACE, "Adding service to cluster", s.UUID)

	err = sm.createPathsForService(s)

	if err != nil {
		return err
	}

	ops := zk.MultiOps{
		Create: []zk.CreateRequest{
			createRequest(path.Join("/regions", s.Region, s.UUID), []byte{}, zk.PermAll, zk.FlagEphemeral),
			createRequest(path.Join("/services", s.Name, s.UUID), []byte{}, zk.PermAll, zk.FlagEphemeral),
			createRequest(path.Join("/services", s.Name, s.Version, s.UUID), []byte{}, zk.PermAll, zk.FlagEphemeral),
			createRequest(path.Join("/hosts", s.ServiceAddr.IPAddress, s.UUID), []byte{}, zk.PermAll, zk.FlagEphemeral),
			createRequest(path.Join("/instances", s.UUID), []byte{}, zk.PermAll, 0),

			createRequest(path.Join("/instances", s.UUID, "registered"), []byte(strconv.FormatBool(s.Registered)), zk.PermAll, zk.FlagEphemeral),
			createRequest(path.Join("/instances", s.UUID, "name"), []byte(s.Name), zk.PermAll, zk.FlagEphemeral),
			createRequest(path.Join("/instances", s.UUID, "version"), []byte(s.Version), zk.PermAll, zk.FlagEphemeral),
			createRequest(path.Join("/instances", s.UUID, "region"), []byte(s.Region), zk.PermAll, zk.FlagEphemeral),
			createRequest(path.Join("/instances", s.UUID, "addr"), []byte(s.ServiceAddr.String()), zk.PermAll, zk.FlagEphemeral),
		},
	}

	err = sm.conn.Multi(ops)

	sm.managedInstances[s.UUID] = s

	return
}

func (cc *ClientCodec) ReadResponseHeader(res *rpc.Response) (err error) {
	log.Println(log.TRACE, "RPC Client Entered: ReadResponseHeader")
	defer log.Println(log.TRACE, "RPC Client Leaving: ReadResponseHeader")

	err = cc.Decoder.Decode(res)

	if err != nil {
		cc.Close()
		log.Println(log.ERROR, "RPC Client Error decoding response header: ", err)
	}

	if err == nil {
		log.Println(log.TRACE, pretty.Sprintf("RPC Client Read ResponseHeader %s %+v", reflect.TypeOf(res), res))
	}

	return
}

func (s *Service) serveAdminRequests() {
	rId := os.Stderr.Fd() + 2
	wId := os.Stderr.Fd() + 3

	pipeReader := os.NewFile(uintptr(rId), "")
	pipeWriter := os.NewFile(uintptr(wId), "")
	s.pipe = daemon.NewPipe(pipeReader, pipeWriter)

	b := make([]byte, daemon.MAX_PIPE_BYTES)
	for {
		n, err := s.pipe.Read(b)

		if err != nil {
			if err != io.EOF {
				log.Printf(log.ERROR, "Error reading from admin pipe "+err.Error())
			} else {
				// We received EOF, ensure we shutdown (if daemon died we could be orphaned)
				s.Shutdown()
			}

			return
		}

		cmd := string(b[:n])
		log.Println(log.TRACE, "Received "+cmd+" from daemon")

		switch cmd {
		case "SHUTDOWN":
			s.Shutdown()
			s.pipe.Write([]byte("ACK"))
			break
		case "REGISTER":
			s.Register()
			s.pipe.Write([]byte("ACK"))
		case "UNREGISTER":
			s.Unregister()
			s.pipe.Write([]byte("ACK"))
		case "LOG DEBUG", "LOG TRACE", "LOG INFO", "LOG WARN", "LOG ERROR", "LOG FATAL", "LOG PANIC":
			parts := strings.Split(cmd, " ")
			log.SetLogLevel(log.LevelFromString(parts[1]))
			log.Println(log.INFO, "Setting log level to "+parts[1])

			s.pipe.Write([]byte("ACK"))
		}
	}
}