func (srv *LspServer) loopServe() {
	var closeAllReply chan error
	for {
		select {
		case p := <-srv.netReadChan:
			srv.handleUdpPacket(p)
		case msg := <-srv.appWriteChan:
			if conn := srv.getConnById(msg.ConnId); conn != nil {
				conn.sendChan <- msg
			}
		case id := <-srv.closeChan:
			if conn := srv.getConnById(id); conn != nil {
				conn.closeChan <- nil
			}
		case closeAllReply = <-srv.closeAllChan:
			srv.stop = true
			for _, v := range srv.connMap {
				v.closeChan <- nil
			}
		case id := <-srv.removeConnChan:
			if conn := srv.getConnById(id); conn != nil {
				delete(srv.connMap, conn.addr.String())
				lsplog.Vlogf(2, "[server] remove connection: %v\n", conn.addr.String())
			}
			if srv.stop && len(srv.connMap) == 0 {
				lsplog.Vlogf(1, "[server] serve stop running\n")
				if closeAllReply != nil {
					closeAllReply <- nil
				}
				return
			}
		}
	}
}

func newLspClient(hostport string, params *LspParams) (*LspClient, error) {
	if params == nil {
		params = &LspParams{5, 2000}
	}
	addr, err := lspnet.ResolveUDPAddr("udp", hostport)
	if lsplog.CheckReport(1, err) {
		return nil, err
	}
	udpConn, err := lspnet.DialUDP("udp", nil, addr)
	if err != nil {
		lsplog.Vlogf(1, "[client] connect to %v failed: %v\n", addr.String(), err)
		return nil, err
	} else {
		lsplog.Vlogf(1, "[client] connected to %v\n", addr.String())
	}
	removeChan := make(chan uint16)
	appReadChan := make(chan *LspMsg)
	conn := newLspConn(params, udpConn, addr, 0, appReadChan, removeChan)
	cli := &LspClient{
		client{
			udpConn:        udpConn,
			addr:           addr,
			conn:           conn,
			netReadChan:    make(chan *LspMsg),
			appWriteChan:   make(chan *LspMsg),
			appReadChan:    appReadChan,
			connIdChan:     make(chan uint16, 1),
			removeConnChan: removeChan,
			closeChan:      make(chan error),
		},
	}
	go cli.loopServe()
	go cli.loopRead()
	return cli, nil
}

// Wait until server has signaled it is done
func (ts *SynchTestSystem) WaitServer() {
	lsplog.Vlogf(6, "Waiting for server\n")
	select {
	case d := <-ts.TChan:
		ts.Tester.Logf("Test reached time limit of %d waiting for server", d)
		ts.Tester.FailNow()
	case <-ts.S2MChan:
	}
	lsplog.Vlogf(6, "Got signal from server\n")
}

// Enable network & then Wait until it signals that it is done
func (ts *SynchTestSystem) SynchNetwork() {
	lsplog.Vlogf(6, "Enabling Network\n")
	ts.M2NChan <- true
	lsplog.Vlogf(6, "Waiting for network\n")
	select {
	case d := <-ts.TChan:
		ts.Tester.Logf("Test reached time limit of %d waiting for network", d)
		ts.Tester.FailNow()
	case <-ts.N2MChan:
	}
	lsplog.Vlogf(6, "Got signal from network\n")
}

// Wait until clients have that they're done
func (ts *SynchTestSystem) WaitClients() {
	lsplog.Vlogf(6, "Waiting for clients\n")
	for i := 0; i < ts.Nclients; i++ {
		select {
		case d := <-ts.TChan:
			ts.Tester.Logf("Test reached time limit of %d waiting for client", d)
			ts.Tester.FailNow()
		case <-ts.C2MChan:
		}
	}

	lsplog.Vlogf(6, "Got signals from all clients\n")
}

func (con *UDPConn) Write(b []byte) (int, error) {
	ncon := con.ncon
	if dropit(writeDropPercent) {
		lsplog.Vlogf(5, "UDP: DROPPING written packet of length %v\n", len(b))
		// Make it look like write was successful
		return len(b), nil
	} else {
		n, err := ncon.Write(b)
		lsplog.Vlogf(5, "UDP: Wrote packet of length %v\n", n)
		return n, err
	}
	return 0, nil
}

func (con *UDPConn) WriteToUDP(b []byte, addr *UDPAddr) (int, error) {
	ncon := con.ncon
	naddr := &net.UDPAddr{IP: addr.IP, Port: addr.Port}
	if dropit(writeDropPercent) {
		lsplog.Vlogf(5, "UDP: DROPPING written packet of length %v\n", len(b))
		// Make it look like write was successful
		return len(b), nil
	} else {
		n, err := ncon.WriteToUDP(b, naddr)
		lsplog.Vlogf(5, "UDP: Wrote packet of length %v", n)
		return n, err
	}
	return 0, nil
}

// Function to coordinate tests
func (ts *SynchTestSystem) Master() {
	// Wait until server and all clients are ready
	ts.WaitServer()
	ts.WaitClients()
	lsplog.Vlogf(4, "Server + all clients started.  Shut off network\n")
	ts.SynchNetwork()
	// Network Off
	// Enable client writes
	if ts.Mode != doservertoclient {
		ts.ReadyClients()
		ts.WaitClients()
	}
	// Do fast close of client.  The calls will not be able to complete
	if ts.Mode == doclienttoserver {
		ts.ReadyClients()
	}
	if ts.Mode != doservertoclient {
		// Turn on network and delay
		ts.SynchNetwork()
		if ts.Mode == doclienttoserver {
			ts.WaitClients()
		}
		ts.SynchNetwork()
		// Network off
		// Enable server reads
		ts.ReadyServer()
		ts.WaitServer()
	}
	// Do server writes
	if ts.Mode != doclienttoserver {
		ts.ReadyServer()
		ts.WaitServer()
	}
	// Do fast close of server.  The calls will not be able to complete
	if ts.Mode != doroundtrip {
		ts.ReadyServer()
	}
	if ts.Mode != doclienttoserver {
		// Turn on network
		ts.SynchNetwork()
		if ts.Mode != doroundtrip {
			// If did quick close, should get responses from server
			ts.WaitServer()
		}
		// Network off again
		ts.SynchNetwork()
		// Enable client reads
		ts.ReadyClients()
		ts.WaitClients()
		// Enable client closes
		ts.ReadyClients()
		ts.WaitClients()
	}
	// Final close by server
	if ts.Mode == doroundtrip {
		ts.ReadyServer()
		ts.WaitServer()
	}
	// Made it!
}

// Have client generate n messages and check that they are echoed.
func (ts *TestSystem) runclient(clienti int) {
	if clienti > ts.NClients {
		ts.Tester.Logf("Invalid client number %d\n", clienti)
		ts.Tester.FailNow()
	}
	cli := ts.Clients[clienti]
	for i := 0; i < ts.NMessages && ts.RunFlag; i++ {
		wt := ts.Rgen.Intn(100)
		werr := cli.Write(i2b(i + wt))
		if werr != nil {
			ts.Tester.Logf("Client write got error '%s'\n",
				werr.Error())
			ts.RunFlag = false
			ts.Tester.FailNow()
		}

		b := cli.Read()
		if b == nil {
			ts.Tester.Logf("Client read got error\n")
			ts.RunFlag = false
			ts.Tester.FailNow()
		}
		v := b2i(b)
		if v != wt+i {
			ts.Tester.Logf("Client got %d.  Expected %d\n",
				v, i+wt)
			ts.RunFlag = false
			ts.Tester.FailNow()
		}
	}
	cli.Close()
	lsplog.Vlogf(0, "Client #%d completed %d messages\n", clienti, ts.NMessages)
	ts.CChan <- 0
}

func (ts *TestSystem) runtest(timeoutms int) {
	lspnet.SetWriteDropPercent(ts.DropPercent)
	if ts.Description != "" {
		fmt.Printf("Testing: %s\n", ts.Description)
	}
	go ts.runserver()
	for i := 0; i < ts.NClients; i++ {
		go ts.runclient(i)
	}
	go ts.runtimeout(timeoutms)
	for i := 0; i < ts.NClients; i++ {
		v := <-ts.CChan
		if v < 0 {
			ts.RunFlag = false
			ts.Tester.Logf("Test timed out after %f secs\n",
				float64(timeoutms)/1000.0)
			ts.Tester.FailNow()
		}
	}
	ts.RunFlag = false
	lsplog.Vlogf(0, "Passed: %d clients, %d messages/client, %.2f maxsleep, %.2f drop rate\n",
		ts.NClients, ts.NMessages,
		float64(ts.MaxSleepMilliseconds)/1000.0,
		float64(ts.DropPercent)/100.0)
	lsplog.SetVerbose(DefaultVerbosity)
	lspnet.SetWriteDropPercent(0)
}

func (conn *lspConn) udpWrite(msg *LspMsg) {
	result, err := json.Marshal(msg)
	if err != nil {
		lsplog.Vlogf(3, "[conn] Marshal failed: %s\n", err.Error())
		return
	}
	switch conn.whichSide {
	case ClientSide:
		_, err = conn.udpConn.Write(result)
	case ServerSide:
		_, err = conn.udpConn.WriteToUDP(result, conn.addr)
	}
	if err != nil {
		lsplog.Vlogf(3, "[conn] udpWrite failed: %s\n", err.Error())
	}
}

func newLspServer(port int, params *LspParams) (*LspServer, error) {
	if params == nil {
		params = &LspParams{5, 2000}
	}
	hostport := fmt.Sprintf("localhost:%v", port)
	addr, err := lspnet.ResolveUDPAddr("udp", hostport)
	if lsplog.CheckReport(1, err) {
		return nil, err
	}
	udpconn, err := lspnet.ListenUDP("udp", addr)
	if lsplog.CheckReport(1, err) {
		return nil, err
	} else {
		lsplog.Vlogf(1, "[server] listen on %v\n", addr.String())
	}
	srv := &LspServer{
		server{
			nextConnId:     1,
			params:         params,
			connMap:        make(map[string]*lspConn),
			udpConn:        udpconn,
			udpAddr:        addr,
			netReadChan:    make(chan *udpPacket),
			appWriteChan:   make(chan *LspMsg),
			appReadChan:    make(chan *LspMsg),
			closeChan:      make(chan uint16),
			closeAllChan:   make(chan chan error),
			removeConnChan: make(chan uint16),
		},
	}
	go srv.loopServe()
	go srv.loopRead()
	return srv, nil
}

func NewLspClient(hostport string, params *LspParams) (*LspClient, error) {
	cli, err := newLspClient(hostport, params)
	if err != nil {
		lsplog.Vlogf(1, "[client] create failure: %v", err)
		return nil, err
	}
	return cli, nil
}

func (con *UDPConn) ReadFromUDP(b []byte) (n int, addr *UDPAddr, err error) {
	var buffer [2000]byte
	ncon := con.ncon
	var naddr *net.UDPAddr
	n, naddr, err = ncon.ReadFromUDP(buffer[0:])
	if dropit(readDropPercent) {
		lsplog.Vlogf(5, "UDP: DROPPING read packet of length %v\n", n)
	} else {
		lsplog.Vlogf(6, "UDP: Read packet of length %v\n", n)
		copy(b, buffer[0:])
	}
	if naddr == nil {
		addr = nil
	} else {
		addr = &UDPAddr{IP: naddr.IP, Port: naddr.Port}
	}
	return n, addr, err
}

// Turn network off and on
func (ts *SynchTestSystem) RunNetwork() {
	// Network initially on
	lspnet.SetWriteDropPercent(0)
	for ts.RunFlag {
		lsplog.Vlogf(4, "Network running.  Waiting for master\n")
		<-ts.M2NChan
		lsplog.Vlogf(4, "Turning off network\n")
		lspnet.SetWriteDropPercent(100)
		ts.N2MChan <- true

		lsplog.Vlogf(4, "Network off.  Waiting for master\n")
		<-ts.M2NChan
		lsplog.Vlogf(4, "Turning network on and delaying\n")
		lspnet.SetWriteDropPercent(0)
		ts.N2MChan <- true
		ts.synchdelay(2.0)
	}
	lsplog.Vlogf(4, "Network handler exiting\n")
}

func (cli *LspClient) loopRead() {
	conn := cli.udpConn
	var buf [2000]byte
	for {
		n, _, err := conn.ReadFromUDP(buf[0:])
		if err != nil {
			lsplog.Vlogf(3, "[client] ReadFromUDP error: %s\n", err.Error())
			continue
		}
		var msg LspMsg
		err = json.Unmarshal(buf[0:n], &msg)
		if err != nil {
			lsplog.Vlogf(3, "[client] Unmarshal error: %s\n", err.Error())
			continue
		}
		cli.netReadChan <- &msg
		lsplog.Vlogf(5, "[client] received udp packet\n")
	}
}

func (conn *lspConn) receive(msg *LspMsg) {
	switch msg.Type {
	case MsgDATA:
		if msg.SeqNum == conn.nextRecvSeqNum {
			conn.recvBuf.Insert(msg)
			conn.nextRecvSeqNum++
			conn.lastAck = genAckMsg(conn.connId, msg.SeqNum)
			// conn.send(conn.lastAck)
		} else {
			lsplog.Vlogf(4, "[conn] ignore data, connId=%v, seqnum=%v, expected=%v\n",
				conn.connId, msg.SeqNum, conn.nextRecvSeqNum)
		}
		conn.udpWrite(conn.lastAck)
	case MsgACK:
		if conn.sendBuf.Empty() {
			lsplog.Vlogf(4, "[conn] ignore ack, nothing to send\n")
			return
		}
		b, _ := conn.sendBuf.Front()
		expect := b.(*LspMsg)
		if msg.SeqNum == expect.SeqNum {
			conn.sendBuf.Remove()
			if msg.SeqNum == 0 {
				conn.connId = msg.ConnId
				lsplog.Vlogf(2, "[conn] connection confirmed, ConnId=%v\n", conn.connId)
			}
			if !conn.sendBuf.Empty() {
				r, _ := conn.sendBuf.Front()
				m := r.(*LspMsg)
				conn.udpWrite(m)
			}
		} else {
			conn.udpWrite(expect)
			lsplog.Vlogf(4, "[conn] ignore ack, ConnId=%v, seqnum=%v, expected=%v\n",
				conn.connId, msg.SeqNum, expect.SeqNum)
		}
		if conn.sendBuf.Empty() && conn.closed {
			conn.writeDone <- nil
		}
	}
}

func (srv *LspServer) handleUdpPacket(p *udpPacket) {
	msg := p.msg
	addr := p.addr
	hostport := addr.String()
	conn := srv.connMap[hostport]
	switch msg.Type {
	case MsgCONNECT:
		if conn != nil {
			lsplog.Vlogf(5, "[server] Duplicate connect request from: %s\n", hostport)
			return
		}
		conn = newLspConn(srv.params, srv.udpConn, addr,
			srv.nextConnId, srv.appReadChan, srv.removeConnChan)
		srv.nextConnId++
		srv.connMap[hostport] = conn
	case MsgDATA:
		conn.recvChan <- msg
	case MsgACK:
		conn.recvChan <- msg
	default:
		lsplog.Vlogf(5, "[server] Invalid packet, hostport=%v\n", hostport)
	}
}

func (srv *LspServer) loopRead() {
	conn := srv.udpConn
	var buf [2000]byte
	for {
		n, addr, err := conn.ReadFromUDP(buf[0:])
		if err != nil {
			lsplog.Vlogf(3, "[server] ReadFromUDP error: %s\n", err.Error())
			continue
		}
		var msg LspMsg
		err = json.Unmarshal(buf[0:n], &msg)
		if err != nil {
			lsplog.Vlogf(3, "[server] Unmarshal error: %s\n", err.Error())
			continue
		}
		packet := &udpPacket{
			msg:  &msg,
			addr: addr,
		}
		srv.netReadChan <- packet
		lsplog.Vlogf(5, "[server] received udp packet\n")
	}
}

func runserver(srv *lsp.LspServer) {
	for {
		// Read from client
		id, payload, rerr := srv.Read()
		if rerr != nil {
			fmt.Printf("Connection %d has died.  Error message %s\n", id, rerr.Error())
		} else {
			s := string(payload)
			lsplog.Vlogf(6, "Connection %d.  Received '%s'\n", id, s)
			payload = []byte(strings.ToUpper(s))
			// Echo back to client
			srv.Write(id, payload)
		}
	}
}