//DeleteProtoNeighbor deletes bgp neighbor for the host
func (self *OfnetBgp) DeleteProtoNeighbor() error {

	/*As a part of delete bgp neighbors
	1) Search for BGP peer and remove from Bgp.
	2) Delete endpoint info for peer
	3) Finally delete all routes learnt on the nexthop bgp port.
	4) Mark the routes learn via json rpc as unresolved
	*/
	log.Infof("Received DeleteProtoNeighbor to delete bgp neighbor %v", self.myBgpPeer)
	client := api.NewGobgpApiClient(self.cc)
	if client == nil {
		log.Errorf("Invalid Gobgpapi client")
		return errors.New("Error creating Gobgpapiclient")
	}
	arg := &api.Arguments{Name: self.myBgpPeer}

	peer, err := client.GetNeighbor(context.Background(), arg)
	if err != nil {
		log.Errorf("GetNeighbor failed ", err)
		return err
	}
	log.Infof("Deleteing Bgp peer from Bgp server")
	p := bgpconf.Neighbor{}
	setNeighborConfigValues(&p)

	p.NeighborAddress = net.ParseIP(peer.Conf.NeighborAddress)
	p.NeighborConfig.NeighborAddress = net.ParseIP(peer.Conf.NeighborAddress)
	p.NeighborConfig.PeerAs = uint32(peer.Conf.PeerAs)
	//FIX ME set ipv6 depending on peerip (for v6 BGP)
	p.AfiSafis.AfiSafiList = []bgpconf.AfiSafi{
		bgpconf.AfiSafi{AfiSafiName: "ipv4-unicast"}}
	self.bgpServer.SetBmpConfig(bgpconf.BmpServers{
		BmpServerList: []bgpconf.BmpServer{},
	})

	self.bgpServer.PeerDelete(p)

	bgpEndpoint := self.agent.getEndpointByIp(net.ParseIP(self.myBgpPeer))
	self.agent.datapath.RemoveEndpoint(bgpEndpoint)
	delete(self.agent.endpointDb, self.myBgpPeer)
	self.myBgpPeer = ""

	uplink, _ := self.agent.ovsDriver.GetOfpPortNo(self.vlanIntf)

	for _, endpoint := range self.agent.endpointDb {
		if endpoint.PortNo == uplink {
			self.agent.datapath.RemoveEndpoint(endpoint)
			if endpoint.EndpointType == "internal" {
				endpoint.PortNo = 0
				self.agent.endpointDb[endpoint.EndpointID] = endpoint
				//We readd unresolved endpoints that were learnt via
				//etcd
				self.agent.datapath.AddEndpoint(endpoint)
			} else if endpoint.EndpointType == "external" {
				delete(self.agent.endpointDb, endpoint.EndpointID)
			}
		}
	}
	return nil
}

func (self *OfnetBgp) InspectProto() (interface{}, error) {
	OfnetBgpInspect := new(OfnetBgpInspect)
	var err error

	client := api.NewGobgpApiClient(self.cc)
	if client == nil {
		log.Errorf("Invalid Gobgpapi client")
		return nil, errors.New("Error creating Gobgpapiclient")
	}

	// Get Bgp info
	arg := &api.Arguments{Name: self.myBgpPeer}
	OfnetBgpInspect.Peer, err = client.GetNeighbor(context.Background(), arg)
	if err != nil {
		log.Errorf("GetNeighbor failed: %v", err)
		return nil, err
	}

	// Get rib info
	table := &api.Table{Type: api.Resource_GLOBAL, Family: uint32(bgp.RF_IPv4_UC)}
	OfnetBgpInspect.Rib, err = client.GetRib(context.Background(), table)
	if err != nil {
		log.Errorf("GetRib failed: %v", err)
		return nil, err
	}

	return OfnetBgpInspect, nil
}

//monitorBest monitors for route updates/changes form peer
func (self *OfnetBgp) monitorBest() {

	client := api.NewGobgpApiClient(self.cc)
	if client == nil {
		log.Errorf("Invalid Gobgpapi client")
		return
	}
	arg := &api.Arguments{
		Resource: api.Resource_GLOBAL,
		Rf:       uint32(bgp.RF_IPv4_UC),
	}

	stream, err := client.MonitorBestChanged(context.Background(), arg)
	if err != nil {
		return
	}

	for {
		dst, err := stream.Recv()
		if err == io.EOF {
			break
		} else if err != nil {
			log.Infof("monitorBest stream ended")
			return
		}
		self.modRibCh <- dst.Paths[0]
	}
	return
}

func NewRootCmd() *cobra.Command {
	cobra.EnablePrefixMatching = true
	rootCmd := &cobra.Command{
		Use: "gobgp",
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			if !globalOpts.GenCmpl {
				conn := connGrpc()
				client = api.NewGobgpApiClient(conn)
			}
		},
		Run: func(cmd *cobra.Command, args []string) {
			cmd.GenBashCompletionFile(globalOpts.BashCmplFile)
		},
	}

	rootCmd.PersistentFlags().StringVarP(&globalOpts.Host, "host", "u", "127.0.0.1", "host")
	rootCmd.PersistentFlags().IntVarP(&globalOpts.Port, "port", "p", 8080, "port")
	rootCmd.PersistentFlags().BoolVarP(&globalOpts.Json, "json", "j", false, "use json format to output format")
	rootCmd.PersistentFlags().BoolVarP(&globalOpts.Debug, "debug", "d", false, "use debug")
	rootCmd.PersistentFlags().BoolVarP(&globalOpts.Quiet, "quiet", "q", false, "use quiet")
	rootCmd.PersistentFlags().BoolVarP(&globalOpts.GenCmpl, "gen-cmpl", "c", false, "generate completion file")
	rootCmd.PersistentFlags().StringVarP(&globalOpts.BashCmplFile, "bash-cmpl-file", "", "gobgp_completion.bash", "bash cmpl filename")

	globalCmd := NewGlobalCmd()
	neighborCmd := NewNeighborCmd()
	vrfCmd := NewVrfCmd()
	policyCmd := NewPolicyCmd()
	monitorCmd := NewMonitorCmd()
	mrtCmd := NewMrtCmd()
	rpkiCmd := NewRPKICmd()
	rootCmd.AddCommand(globalCmd, neighborCmd, vrfCmd, policyCmd, monitorCmd, mrtCmd, rpkiCmd)
	return rootCmd
}

// monitorPeer is used to monitor the bgp peer state
func (self *OfnetBgp) monitorPeer() {

	var oldAdminState, oldState string

	client := api.NewGobgpApiClient(self.cc)
	if client == nil {
		log.Errorf("Invalid Gobgpapi client")
		return
	}
	arg := &api.Arguments{}

	stream, err := client.MonitorPeerState(context.Background(), arg)
	if err != nil {
		log.Errorf("MonitorPeerState failed ", err)
		return
	}
	for {
		s, err := stream.Recv()
		if err == io.EOF {
			break
		} else if err != nil {
			log.Warnf("MonitorPeerState stream ended :")
			break
		}
		fmt.Printf("[NEIGH] %s fsm: %s admin: %s\n", s.Conf.NeighborAddress,
			s.Info.BgpState, s.Info.AdminState)
		if oldState == "BGP_FSM_ESTABLISHED" && oldAdminState == "ADMIN_STATE_UP" {
			uplink, _ := self.agent.ovsDriver.GetOfpPortNo(self.vlanIntf)
			/*If the state changed from being established to idle or active:
			   1) delete all endpoints learnt via bgp Peer
				 2) mark routes pointing to the bgp nexthop as unresolved
				 3) mark the bgp peer reachbility as unresolved
			*/
			endpoint := self.agent.getEndpointByIp(net.ParseIP(self.myBgpPeer))
			self.agent.datapath.RemoveEndpoint(endpoint)
			endpoint.PortNo = 0
			self.agent.endpointDb[endpoint.EndpointID] = endpoint
			self.agent.datapath.AddEndpoint(endpoint)

			for _, endpoint = range self.agent.endpointDb {
				if endpoint.PortNo == uplink {
					self.agent.datapath.RemoveEndpoint(endpoint)
					if endpoint.EndpointType == "internal" {
						endpoint.PortNo = 0
						self.agent.endpointDb[endpoint.EndpointID] = endpoint
						//We readd unresolved endpoints that were learnt via
						//json rpc
						self.agent.datapath.AddEndpoint(endpoint)
					} else if endpoint.EndpointType == "external" {
						delete(self.agent.endpointDb, endpoint.EndpointID)
					}
				}
			}
		}
		oldState = s.Info.BgpState
		oldAdminState = s.Info.AdminState
	}
	return
}

//DeleteLocalProtoRoute withdraws local endpoints from protocol RIB
func (self *OfnetBgp) DeleteLocalProtoRoute(pathInfo *OfnetProtoRouteInfo) error {

	log.Infof("Received DeleteLocalProtoRoute to withdraw local endpoint to protocol RIB: %v", pathInfo)

	path := &api.Path{
		Pattrs: make([][]byte, 0),
	}

	//form appropraite path attributes for path to be withdrawn
	nlri := bgp.NewIPAddrPrefix(32, pathInfo.localEpIP)
	path.Nlri, _ = nlri.Serialize()
	origin, _ := bgp.NewPathAttributeOrigin(bgp.BGP_ORIGIN_ATTR_TYPE_EGP).Serialize()
	path.Pattrs = append(path.Pattrs, origin)
	aspathParam := []bgp.AsPathParamInterface{bgp.NewAs4PathParam(2, []uint32{self.myBgpAs})}
	aspath, _ := bgp.NewPathAttributeAsPath(aspathParam).Serialize()
	path.Pattrs = append(path.Pattrs, aspath)
	n, _ := bgp.NewPathAttributeNextHop(pathInfo.nextHopIP).Serialize()
	path.Pattrs = append(path.Pattrs, n)
	path.IsWithdraw = true
	name := ""

	arg := &api.ModPathArguments{
		Resource: api.Resource_GLOBAL,
		Name:     name,
		Paths:    []*api.Path{path},
	}

	//send arguement stream
	client := api.NewGobgpApiClient(self.cc)
	if client == nil {
		log.Errorf("Gobgpapi stream invalid")
		return nil
	}

	stream, err := client.ModPath(context.Background())
	if err != nil {
		log.Errorf("Fail to enforce Modpathi", err)
		return err
	}

	err = stream.Send(arg)
	if err != nil {
		log.Errorf("Failed to send strean", err)
		return err
	}
	stream.CloseSend()

	res, e := stream.CloseAndRecv()
	if e != nil {
		log.Errorf("Falied toclose stream ")
		return e
	}

	if res.Code != api.Error_SUCCESS {
		return fmt.Errorf("error: code: %d, msg: %s", res.Code, res.Msg)
	}
	return nil
}

func (b *BgpRouteManager) StartMonitoring() error {
	err := cleanExistingRoutes(b.ethIface)
	if err != nil {
		log.Infof("Error cleaning old routes: %s", err)
	}
	bgpCache := &RibCache{
		BgpTable: make(map[string]*RibLocal),
	}
	timeout := grpc.WithTimeout(time.Second)
	conn, err := grpc.Dial("127.0.0.1:8080", timeout, grpc.WithBlock(), grpc.WithInsecure())
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()
	b.bgpgrpcclient = api.NewGobgpApiClient(conn)
	RibCh := make(chan *api.Path)
	go b.monitorBestPath(RibCh)

	log.Info("Initialization complete, now monitoring BGP for new routes..")
	for {
		select {
		case p := <-RibCh:
			monitorUpdate, err := bgpCache.handleBgpRibMonitor(p)

			if err != nil {
				log.Errorf("error processing bgp update [ %s ]", err)
			}
			if monitorUpdate.IsLocal != true {
				if p.IsWithdraw {
					monitorUpdate.IsWithdraw = true
					log.Infof("BGP update has [ withdrawn ] the IP prefix [ %s ]", monitorUpdate.BgpPrefix.String())
					// If the bgp update contained a withdraw, remove the local netlink route for the remote endpoint
					err = delNetlinkRoute(monitorUpdate.BgpPrefix, monitorUpdate.NextHop, b.ethIface)
					if err != nil {
						log.Errorf("Error removing learned bgp route [ %s ]", err)
					}
				} else {
					monitorUpdate.IsWithdraw = false
					b.learnedRoutes = append(b.learnedRoutes, *monitorUpdate)
					log.Debugf("Learned routes: %v ", monitorUpdate)

					err = addNetlinkRoute(monitorUpdate.BgpPrefix, monitorUpdate.NextHop, b.ethIface)
					if err != nil {
						log.Debugf("Add route results [ %s ]", err)
					}

					log.Infof("Updated the local prefix cache from the newly learned BGP update:")
					for n, entry := range b.learnedRoutes {
						log.Debugf("%d - %+v", n+1, entry)
					}
				}
			}
			log.Debugf("Verbose update details: %s", monitorUpdate)
		}
	}
}

func (b *BgpRouteManager) monitorBestPath(RibCh chan *api.Path) error {
	timeout := grpc.WithTimeout(time.Second)
	conn, err := grpc.Dial("127.0.0.1:8080", timeout, grpc.WithBlock(), grpc.WithInsecure())
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()
	client := api.NewGobgpApiClient(conn)

	arg := &api.Arguments{
		Resource: api.Resource_GLOBAL,
		Rf:       uint32(bgp.RF_IPv4_UC),
	}
	err = func() error {
		stream, err := client.GetRib(context.Background(), arg)
		if err != nil {
			return err
		}
		for {
			dst, err := stream.Recv()
			if err == io.EOF {
				break
			} else if err != nil {
				return err
			}
			for _, p := range dst.Paths {
				if p.Best {
					RibCh <- p
					break
				}
			}
		}
		return nil
	}()

	if err != nil {
		return err
	}
	stream, err := client.MonitorBestChanged(context.Background(), arg)
	if err != nil {
		return err
	}
	for {
		dst, err := stream.Recv()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}
		RibCh <- dst.Paths[0]
	}
	return nil
}

func (b *BgpRouteManager) monitorBestPath(RibCh chan *api.Path) error {
	timeout := grpc.WithTimeout(time.Second)
	conn, err := grpc.Dial(GrcpServer, timeout, grpc.WithBlock(), grpc.WithInsecure())
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()
	client := api.NewGobgpApiClient(conn)

	table := &api.Table{
		Type:   api.Resource_GLOBAL,
		Family: uint32(bgp.RF_IPv4_UC),
		Name:   "",
	}
	arg := &api.Arguments{
		Resource: api.Resource_GLOBAL,
		Family:   uint32(bgp.RF_IPv4_UC),
	}

	err = func() error {
		rib, err := client.GetRib(context.Background(), table)
		if err != nil {
			return err
		}
		for _, d := range rib.Destinations {
			for _, p := range d.Paths {
				if p.Best {
					RibCh <- p
					break
				}
			}
		}
		return nil
	}()

	if err != nil {
		return err
	}
	stream, err := client.MonitorBestChanged(context.Background(), arg)
	if err != nil {
		return err
	}
	for {
		dst, err := stream.Recv()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}
		RibCh <- dst.Paths[0]
	}
	return nil
}

func NewGoBGPClient(target string, opts ...grpc.DialOption) (*GoBGPClient, error) {
	if target == "" {
		target = ":50051"
	}
	if len(opts) == 0 {
		opts = defaultGRPCOptions()
	}
	conn, err := grpc.Dial(target, opts...)
	if err != nil {
		return nil, err
	}
	cli := api.NewGobgpApiClient(conn)
	return &GoBGPClient{conn: conn, cli: cli}, nil
}

func TestOfnetBgpPeerAddDelete(t *testing.T) {

	neighborAs := "500"
	peer := "50.1.1.2"
	routerIP := "50.1.1.1/24"
	as := "65002"
	//Add Bgp neighbor and check if it is successful

	for i := 0; i < NUM_VLRTR_AGENT; i++ {
		err := vlrtrAgents[i].AddBgp(routerIP, as, neighborAs, peer)
		if err != nil {
			t.Errorf("Error adding Bgp Neighbor: %v", err)
			return
		}

		timeout := grpc.WithTimeout(time.Second)
		conn, err := grpc.Dial("127.0.0.1:8080", timeout, grpc.WithBlock(), grpc.WithInsecure())
		if err != nil {
			t.Fatal(err)
		}
		defer conn.Close()
		client := api.NewGobgpApiClient(conn)
		if client == nil {
			t.Errorf("GoBgpApiclient is invalid")
		}
		arg := &api.Arguments{Name: peer}

		//Check if neighbor is added to bgp server
		bgpPeer, err := client.GetNeighbor(context.Background(), arg)
		if err != nil {
			t.Errorf("GetNeighbor failed: %v", err)
			return
		}

		//Delete BGP neighbor
		err = vlrtrAgents[i].DeleteBgp()
		if err != nil {
			t.Errorf("Error Deleting Bgp Neighbor: %v", err)
			return
		}

		//Check if neighbor is added to bgp server
		bgpPeer, err = client.GetNeighbor(context.Background(), arg)
		if bgpPeer != nil {
			t.Errorf("Neighbor is not deleted: %v", err)
			return
		}
	}
}

//DeleteLocalProtoRoute withdraws local endpoints from protocol RIB
func (self *OfnetBgp) DeleteLocalProtoRoute(pathInfo *OfnetProtoRouteInfo) error {

	log.Infof("Received DeleteLocalProtoRoute to withdraw local endpoint to protocol RIB: %v", pathInfo)

	path := &api.Path{
		Pattrs: make([][]byte, 0),
	}

	//form appropraite path attributes for path to be withdrawn
	nlri := bgp.NewIPAddrPrefix(32, pathInfo.localEpIP)
	path.Nlri, _ = nlri.Serialize()
	origin, _ := bgp.NewPathAttributeOrigin(bgp.BGP_ORIGIN_ATTR_TYPE_EGP).Serialize()
	path.Pattrs = append(path.Pattrs, origin)
	aspathParam := []bgp.AsPathParamInterface{bgp.NewAs4PathParam(2, []uint32{self.myBgpAs})}
	aspath, _ := bgp.NewPathAttributeAsPath(aspathParam).Serialize()
	path.Pattrs = append(path.Pattrs, aspath)
	n, _ := bgp.NewPathAttributeNextHop(pathInfo.nextHopIP).Serialize()
	path.Pattrs = append(path.Pattrs, n)
	path.IsWithdraw = true
	//paths := []*api.Path{path}
	arg := &api.DeletePathRequest{
		Resource: api.Resource_GLOBAL,
		Path:     path,
	}

	//send arguement stream
	client := api.NewGobgpApiClient(self.cc)
	if client == nil {
		log.Errorf("Gobgpapi stream invalid")
		return nil
	}

	_, err := client.DeletePath(context.Background(), arg)

	if err != nil {
		log.Errorf("Fail to enforce Modpathi: %v", err)
		return err
	}

	return nil
}

func (m *OpsManager) GobgpMonitor(target string) {
	time.Sleep(time.Duration(time.Second * 2))

	conn, err := grpc.Dial(target, grpc.WithTimeout(time.Second), grpc.WithBlock(), grpc.WithInsecure())
	if err != nil {
		log.Fatal(err)
	}
	cli := api.NewGobgpApiClient(conn)
	stream, err := cli.MonitorRib(context.Background(), &api.Table{
		Type:   api.Resource_GLOBAL,
		Family: uint32(bgp.RF_IPv4_UC),
	})
	for {
		d, err := stream.Recv()
		bPath := d.Paths[0]
		if bPath.IsFromExternal && !bPath.IsWithdraw {
			continue
		}
		p, err := cmd.ApiStruct2Path(bPath)
		if err != nil {
			log.WithFields(log.Fields{
				"Topic": "openswitch",
				"Type":  "MonitorRequest",
				"Error": err,
			}).Error("failed parse path of gobgp")
		}
		o, err := m.TransactPreparation(p)
		if err != nil {
			log.WithFields(log.Fields{
				"Topic": "openswitch",
				"Type":  "Monitor",
				"Error": err,
			}).Error("failed transact preparation of ops")
		}
		m.opsCh <- o
	}
}

func (m *OpsManager) OpsServe(target string) error {
	initial, err := m.ops.MonitorAll(TARGET_TABLE, "")
	if err != nil {
		return err
	}
	go func() {
		m.opsUpdateCh <- initial
	}()
	conn, err := grpc.Dial(target, grpc.WithTimeout(time.Second), grpc.WithBlock(), grpc.WithInsecure())
	if err != nil {
		log.Fatal(err)
	}
	cli := api.NewGobgpApiClient(conn)
	for {
		select {
		case updates := <-m.opsUpdateCh:
			m.populateCache(*updates)
			if t, ok := updates.Updates["VRF"]; ok {
				m.handleVrfUpdate(cli, t)
			}
			if t, ok := updates.Updates["BGP_Router"]; ok {
				m.handleBgpRouterUpdate(cli, t)
			}
			if t, ok := updates.Updates["BGP_Neighbor"]; ok {
				m.handleNeighborUpdate(cli, t)
			}
			if t, ok := updates.Updates["BGP_Route"]; ok {
				m.handleRouteUpdate(cli, t)
			}
		case r := <-m.opsCh:
			if err := m.Transact(r.operations); err != nil {
			}
		}
	}
	return nil
}

//AddLocalProtoRoute is used to add local endpoint to the protocol RIB
func (self *OfnetBgp) AddLocalProtoRoute(pathInfo *OfnetProtoRouteInfo) error {
	log.Infof("Received AddLocalProtoRoute to add local endpoint to protocol RIB: %v", pathInfo)
	if self.routerIP == "" {
		//ignoring populating to the bgp rib because
		//Bgp is not configured.
		return nil
	}

	path := &api.Path{
		Pattrs:    make([][]byte, 0),
		SourceAsn: self.myBgpAs,
	}

	aspathParam := []bgp.AsPathParamInterface{bgp.NewAs4PathParam(2, []uint32{self.myBgpAs})}
	aspath, _ := bgp.NewPathAttributeAsPath(aspathParam).Serialize()
	path.Pattrs = append(path.Pattrs, aspath)

	// form the path structure with appropriate path attributes
	nlri := bgp.NewIPAddrPrefix(32, pathInfo.localEpIP)
	path.Nlri, _ = nlri.Serialize()
	origin, _ := bgp.NewPathAttributeOrigin(bgp.BGP_ORIGIN_ATTR_TYPE_EGP).Serialize()
	path.Pattrs = append(path.Pattrs, origin)
	log.Infof("Received AddLocalProtoRoute aspath: %v", aspath)
	mpls, err := bgp.ParseMPLSLabelStack("3")
	if err != nil {
		return nil
	}
	var nlri2 bgp.AddrPrefixInterface
	nlri2 = bgp.NewLabeledIPAddrPrefix(32, pathInfo.localEpIP, *mpls)
	mpreach, _ := bgp.NewPathAttributeMpReachNLRI(pathInfo.nextHopIP, []bgp.AddrPrefixInterface{nlri2}).Serialize()
	log.Infof("Received AddLocalProtoRoute nlri2: %v, mpreach: %v", nlri2, mpreach)
	path.Pattrs = append(path.Pattrs, mpreach)
	//path.Pattrs = append(path.Pattrs, n)

	arg := &api.AddPathRequest{
		Resource: api.Resource_GLOBAL,
		VrfId:    "default",
		Path:     path,
	}
	log.Infof("BGP - arg %v ", arg)
	// add routes
	/*
	   	prefix := pathInfo.localEpIP
	   	log.Infof("BGP - Prefix %v " , prefix )
	       	path2, err := cmd.ParsePath(bgp.RF_IPv4_MPLS, []string{"10.200.1.25/32", "3"})
	   	log.Infof("BGP - Path2 %v " , path2 )
	   	log.Infof("BGP - Err %v " , err )
	       	req := bgpserver.NewGrpcRequest(bgpserver.REQ_ADD_PATH, "", bgp.RouteFamily(0), &api.AddPathRequest{
	           	Resource: api.Resource_GLOBAL,
	           	Path:     path2,
	       	})
	       	self.bgpServer.GrpcReqCh <- req
	       	res := <-req.ResponseCh
	       	if err := res.Err(); err != nil {
	           	log.Fatal(err)
	   		log.Infof("BGP - Err %v " , err)
	       	}
	*/
	//send arguement stream
	client := api.NewGobgpApiClient(self.cc)
	if client == nil {
		log.Infof("Gobgpapi stream invalid")
		return nil
	}
	stream, err := client.AddPath(context.Background(), arg)
	if err != nil {
		log.Infof("Fail to enforce Modpath: %v %v", err, stream)
		return err
	}

	return nil
}

func (b *BgpRouteManager) StartMonitoring() error {
	err := cleanExistingRoutes(b.ethIface)
	if err != nil {
		log.Infof("Error cleaning old routes: %s", err)
	}
	bgpCache := &RibCache{
		BgpTable: make(map[string]*RibLocal),
	}
	timeout := grpc.WithTimeout(time.Second)
	conn, err := grpc.Dial(GrcpServer, timeout, grpc.WithBlock(), grpc.WithInsecure())
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()
	b.bgpgrpcclient = api.NewGobgpApiClient(conn)
	g, err := b.bgpgrpcclient.GetGlobalConfig(context.Background(), &api.Arguments{})
	if err != nil {
		b.autoconfig = true
		log.Info("Config file is not detectd. Configuration with hostdiscovery and grpc")
	} else {
		b.autoconfig = false
		log.Infof("Config file is detectd. Global config %v", g)
		go b.monitorBestPath(b.RibCh)
	}
	log.Info("Initialization complete")
	for {
		select {
		case p := <-b.RibCh:
			monitorUpdate, err := bgpCache.handleBgpRibMonitor(p)

			if err != nil {
				log.Errorf("error processing bgp update [ %s ]", err)
			}
			if monitorUpdate.IsLocal != true {
				if p.IsWithdraw {
					monitorUpdate.IsWithdraw = true
					log.Infof("BGP update has [ withdrawn ] the IP prefix [ %s ]", monitorUpdate.BgpPrefix.String())
					// If the bgp update contained a withdraw, remove the local netlink route for the remote endpoint
					err = delNetlinkRoute(monitorUpdate.BgpPrefix, monitorUpdate.NextHop, b.ethIface)
					if err != nil {
						log.Errorf("Error removing learned bgp route [ %s ]", err)
					}
				} else {
					monitorUpdate.IsWithdraw = false
					b.learnedRoutes = append(b.learnedRoutes, *monitorUpdate)
					log.Debugf("Learned routes: %v ", monitorUpdate)

					err = addNetlinkRoute(monitorUpdate.BgpPrefix, monitorUpdate.NextHop, b.ethIface)
					if err != nil {
						log.Debugf("Error Adding route results [ %s ]", err)
					}

					log.Infof("Updated the local prefix cache from the newly learned BGP update:")
					for n, entry := range b.learnedRoutes {
						log.Debugf("%d - %+v", n+1, entry)
					}
				}
			}
			log.Debugf("Verbose update details: %s", monitorUpdate)

		case arg := <-b.ModPeerCh:
			_, err := b.bgpgrpcclient.ModNeighbor(context.Background(), arg)
			log.Debugf("Mod Peer with grpc %v", arg)
			if err != nil {
				return err
			}

		case path := <-b.ModPathCh:
			n, _ := bgp.NewPathAttributeNextHop("0.0.0.0").Serialize()
			path.Pattrs = append(path.Pattrs, n)
			origin, _ := bgp.NewPathAttributeOrigin(bgp.BGP_ORIGIN_ATTR_TYPE_IGP).Serialize()
			path.Pattrs = append(path.Pattrs, origin)
			arg := &api.ModPathArguments{
				Operation: api.Operation_ADD,
				Resource:  api.Resource_GLOBAL,
				Name:      "",
				Path:      path,
			}
			log.Debugf("Mod Path with grcp %v", arg)
			_, err := b.bgpgrpcclient.ModPath(context.Background(), arg)
			if err != nil {
				return err
			}
		}
	}
}

/*
Bgp serve routine does the following:
1) Creates inb01 router port
2) Add MyBgp endpoint
3) Kicks off routines to monitor route updates and peer state
*/
func (self *OfnetBgp) StartProtoServer(routerInfo *OfnetProtoRouterInfo) error {
	log.Infof("Starting the Bgp Server with %v", routerInfo)
	//go routine to start gobgp server
	var len uint
	var err error
	self.routerIP, len, err = ParseCIDR(routerInfo.RouterIP)
	as, _ := strconv.Atoi(routerInfo.As)
	self.myBgpAs = uint32(as)

	self.modRibCh = make(chan *api.Path, 16)
	self.advPathCh = make(chan *api.Path, 16)

	timeout := grpc.WithTimeout(time.Second)
	conn, err := grpc.Dial("127.0.0.1:8080", timeout, grpc.WithBlock(), grpc.WithInsecure())
	if err != nil {
		log.Fatal(err)
	}
	self.cc = conn
	defer self.cc.Close()

	client := api.NewGobgpApiClient(self.cc)
	if client == nil {
		log.Errorf("Invalid Gobgpapi client")
		return errors.New("Error creating Gobgpapiclient")
	}
	path := &api.Path{
		Pattrs: make([][]byte, 0),
	}

	path.Nlri, _ = bgp.NewIPAddrPrefix(uint8(32), self.routerIP).Serialize()
	n, _ := bgp.NewPathAttributeNextHop("0.0.0.0").Serialize()
	path.Pattrs = append(path.Pattrs, n)
	origin, _ := bgp.NewPathAttributeOrigin(bgp.BGP_ORIGIN_ATTR_TYPE_INCOMPLETE).Serialize()
	path.Pattrs = append(path.Pattrs, origin)

	log.Debugf("Creating the loopback port ")
	err = self.agent.ovsDriver.CreatePort(self.intfName, "internal", 1)
	if err != nil {
		log.Errorf("Error creating the port", err)
	}

	intfIP := fmt.Sprintf("%s/%d", self.routerIP, len)
	log.Debugf("Creating inb01 with ", intfIP)
	ofPortno, _ := self.agent.ovsDriver.GetOfpPortNo(self.intfName)

	link, err := netlink.LinkByName(self.intfName)
	if err != nil {
		log.Errorf("error finding link by name %v", self.intfName)
		return err
	}
	linkIP, err := netlink.ParseAddr(intfIP)
	if err != nil {
		log.Errorf("invalid ip ", intfIP)
	}
	netlink.AddrAdd(link, linkIP)

	if link == nil || ofPortno == 0 {
		log.Errorf("Error fetching %v information", self.intfName, link, ofPortno)
		return errors.New("Unable to fetch inb01 info")
	}

	intf, _ := net.InterfaceByName(self.intfName)

	epreg := &OfnetEndpoint{
		EndpointID:   self.routerIP,
		EndpointType: "internal-bgp",
		IpAddr:       net.ParseIP(self.routerIP),
		IpMask:       net.ParseIP("255.255.255.255"),
		VrfId:        0,                          // FIXME set VRF correctly
		MacAddrStr:   intf.HardwareAddr.String(), //link.Attrs().HardwareAddr.String(),
		Vlan:         1,
		PortNo:       ofPortno,
		Timestamp:    time.Now(),
	}

	// Add the endpoint to local routing table
	self.agent.endpointDb[self.routerIP] = epreg
	self.agent.localEndpointDb[epreg.PortNo] = epreg
	fmt.Println(epreg)
	err = self.agent.datapath.AddLocalEndpoint(*epreg)

	//Add bgp router id as well
	bgpGlobalCfg := &bgpconf.Global{}
	setDefaultGlobalConfigValues(bgpGlobalCfg)
	bgpGlobalCfg.GlobalConfig.RouterId = net.ParseIP(self.routerIP)
	bgpGlobalCfg.GlobalConfig.As = self.myBgpAs
	self.bgpServer.SetGlobalType(*bgpGlobalCfg)

	self.advPathCh <- path

	//monitor route updates from peer
	go self.monitorBest()
	//monitor peer state
	go self.monitorPeer()
//.........这里部分代码省略.........

/*
Bgp serve routine does the following:
1) Creates inb01 router port
2) Add MyBgp endpoint
3) Kicks off routines to monitor route updates and peer state
*/
func (self *OfnetBgp) StartProtoServer(routerInfo *OfnetProtoRouterInfo) error {
	log.Infof("Starting the Bgp Server with %v", routerInfo)
	//go routine to start gobgp server
	var len uint
	var err error
	self.routerIP, len, err = ParseCIDR(routerInfo.RouterIP)
	as, _ := strconv.Atoi(routerInfo.As)
	self.myBgpAs = uint32(as)
	log.Infof("Starting the Bgp Server for AS %v", self.myBgpAs)

	self.modRibCh = make(chan *api.Path, 16)
	self.advPathCh = make(chan *api.Path, 16)

	timeout := grpc.WithTimeout(time.Second)
	conn, err := grpc.Dial("127.0.0.1:50051", timeout, grpc.WithBlock(), grpc.WithInsecure())
	if err != nil {
		log.Infof("GRPC timeout")
		log.Fatal(err)
	}
	self.cc = conn
	defer self.cc.Close()

	client := api.NewGobgpApiClient(self.cc)
	if client == nil {
		log.Errorf("Invalid Gobgpapi client")
		return errors.New("Error creating Gobgpapiclient")
	}
	path := &api.Path{
		Pattrs: make([][]byte, 0),
	}

	path.Nlri, _ = bgp.NewIPAddrPrefix(uint8(32), self.routerIP).Serialize()
	n, _ := bgp.NewPathAttributeNextHop("0.0.0.0").Serialize()
	path.Pattrs = append(path.Pattrs, n)
	origin, _ := bgp.NewPathAttributeOrigin(bgp.BGP_ORIGIN_ATTR_TYPE_INCOMPLETE).Serialize()
	path.Pattrs = append(path.Pattrs, origin)

	log.Infof("Creating the loopback port ")
	err = self.agent.ovsDriver.CreatePort(self.intfName, "internal", 1)
	if err != nil {
		log.Errorf("Error creating the port: %v", err)
	}
	defer self.agent.ovsDriver.DeletePort(self.intfName)

	intfIP := fmt.Sprintf("%s/%d", self.routerIP, len)
	log.Infof("Creating inb01 with ", intfIP)
	ofPortno, _ := self.agent.ovsDriver.GetOfpPortNo(self.intfName)

	link, err := netlink.LinkByName(self.intfName)
	if err != nil {
		log.Errorf("error finding link by name %v", self.intfName)
		return err
	}
	linkIP, err := netlink.ParseAddr(intfIP)
	if err != nil {
		log.Errorf("invalid ip: %s", intfIP)
		return err
	}
	netlink.AddrAdd(link, linkIP)
	netlink.LinkSetUp(link)
	if link == nil || ofPortno == 0 {
		log.Errorf("Error fetching %v/%v/%v information", self.intfName, link, ofPortno)
		return errors.New("Unable to fetch inb01 info")
	}

	intf, _ := net.InterfaceByName(self.intfName)
	log.Infof("routerIP ", self.routerIP)
	epid := self.agent.getEndpointIdByIpVrf(net.ParseIP(self.routerIP), "default")
	log.Infof("Create VRF ")
	_, ok := self.agent.createVrf("default")
	if !ok {
		log.Errorf("Error Creating default vrf for Bgp")
		return errors.New("Error creating default vrf")
	}

	epreg := &OfnetEndpoint{
		EndpointID:   epid,
		EndpointType: "internal-bgp",
		IpAddr:       net.ParseIP(self.routerIP),
		IpMask:       net.ParseIP("255.255.255.255"),
		Vrf:          "default",                  // FIXME set VRF correctly
		MacAddrStr:   intf.HardwareAddr.String(), //link.Attrs().HardwareAddr.String(),
		Vlan:         1,
		PortNo:       ofPortno,
		Timestamp:    time.Now(),
	}

	// Add the endpoint to local routing table

	self.agent.endpointDb[epid] = epreg
	//log.Infof("self.agent.endpointDb[epid] %v " , self.agent.endpointDb[epid] )
	self.agent.localEndpointDb[epreg.PortNo] = epreg
	fmt.Println(epreg)
	err = self.agent.datapath.AddLocalEndpoint(*epreg)
//.........这里部分代码省略.........

/*
Bgp serve routine does the following:
1) Creates inb01 router port
2) Add MyBgp endpoint
3) Kicks off routines to monitor route updates and peer state
*/
func (self *OfnetBgp) StartProtoServer(routerInfo OfnetProtoRouterInfo) error {
	time.Sleep(5 * time.Second)
	self.agent.WaitForSwitchConnection()

	self.modRibCh = make(chan *api.Path, 16)
	self.advPathCh = make(chan *api.Path, 16)

	timeout := grpc.WithTimeout(time.Second)
	conn, err := grpc.Dial("127.0.0.1:8080", timeout, grpc.WithBlock(), grpc.WithInsecure())
	if err != nil {
		log.Fatal(err)
	}
	self.cc = conn
	defer self.cc.Close()

	client := api.NewGobgpApiClient(self.cc)
	if client == nil {
		log.Errorf("Invalid Gobgpapi client")
		return errors.New("Error creating Gobgpapiclient")
	}
	path := &api.Path{
		Pattrs: make([][]byte, 0),
	}

	if len(routerInfo.RouterIP) == 0 {
		log.Errorf("Invalid router IP. Bgp service aborted")
		return errors.New("Invalid router IP")
	}
	path.Nlri, _ = bgp.NewIPAddrPrefix(uint8(32), routerInfo.RouterIP).Serialize()
	n, _ := bgp.NewPathAttributeNextHop("0.0.0.0").Serialize()
	path.Pattrs = append(path.Pattrs, n)
	origin, _ := bgp.NewPathAttributeOrigin(bgp.BGP_ORIGIN_ATTR_TYPE_INCOMPLETE).Serialize()
	path.Pattrs = append(path.Pattrs, origin)

	err = self.agent.ovsDriver.CreatePort("inb01", "internal", 1)
	if err != nil {
		log.Errorf("Error creating the port", err)
	}

	cmd := exec.Command("ifconfig", "inb01", routerInfo.RouterIP+"/24")
	cmd.Run()

	intf, _ := net.InterfaceByName("inb01")
	ofPortno, _ := self.agent.ovsDriver.GetOfpPortNo("inb01")

	if intf == nil || ofPortno == 0 {
		log.Errorf("Error fetching inb01 information", intf, ofPortno)
		return errors.New("Unable to fetch inb01 info")
	}

	epreg := &OfnetEndpoint{
		EndpointID:   routerInfo.RouterIP,
		EndpointType: "internal-bgp",
		IpAddr:       net.ParseIP(routerInfo.RouterIP),
		IpMask:       net.ParseIP("255.255.255.255"),
		VrfId:        0,                          // FIXME set VRF correctly
		MacAddrStr:   intf.HardwareAddr.String(), //link.Attrs().HardwareAddr.String(),
		Vlan:         1,
		PortNo:       ofPortno,
		Timestamp:    time.Now(),
	}
	// Add the endpoint to local routing table
	self.agent.endpointDb[routerInfo.RouterIP] = epreg
	self.agent.localEndpointDb[epreg.PortNo] = epreg
	fmt.Println(epreg)
	err = self.agent.datapath.AddLocalEndpoint(*epreg)

	//Add bgp router id as well
	bgpGlobalCfg := &bgpconf.Global{}
	setDefaultGlobalConfigValues(bgpGlobalCfg)
	bgpGlobalCfg.GlobalConfig.RouterId = net.ParseIP(routerInfo.RouterIP)
	bgpGlobalCfg.GlobalConfig.As = 65002
	self.bgpServer.SetGlobalType(*bgpGlobalCfg)

	self.advPathCh <- path

	//monitor route updates from peer
	go self.monitorBest()
	//monitor peer state
	go self.monitorPeer()

	for {
		select {
		case p := <-self.modRibCh:
			err = self.modRib(p)
			if err != nil {
				log.Error("failed to mod rib: ", err)
			}
		}
	}
}

func (b *BgpRouteManager) StartMonitoring() error {
	err := cleanExistingRoutes(b.ethIface)
	if err != nil {
		log.Infof("Error cleaning old routes: %s", err)
	}
	bgpCache := &RibCache{
		BgpTable: make(map[string]*RibLocal),
	}
	timeout := grpc.WithTimeout(time.Second)
	conn, err := grpc.Dial("127.0.0.1:8080", timeout, grpc.WithBlock(), grpc.WithInsecure())
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()
	b.bgpgrpcclient = api.NewGobgpApiClient(conn)
	RibCh := make(chan *api.Path)
	go b.monitorBestPath(RibCh)
	stopCh := make(<-chan struct{})
	events, err := b.kv.WatchTree(b.neighborkey, stopCh)
	if err != nil {
		log.Errorf("Error trying to WatchTree: %v", err)
	}
	b.SetBgpConfig()

	log.Info("Initialization complete, now monitoring BGP for new routes..")
	for {
		select {
		case p := <-RibCh:
			monitorUpdate, err := bgpCache.handleBgpRibMonitor(p)

			if err != nil {
				log.Errorf("error processing bgp update [ %s ]", err)
			}
			if monitorUpdate.IsLocal != true {
				if p.IsWithdraw {
					monitorUpdate.IsWithdraw = true
					log.Infof("BGP update has [ withdrawn ] the IP prefix [ %s ]", monitorUpdate.BgpPrefix.String())
					// If the bgp update contained a withdraw, remove the local netlink route for the remote endpoint
					err = delNetlinkRoute(monitorUpdate.BgpPrefix, monitorUpdate.NextHop, b.ethIface)
					if err != nil {
						log.Errorf("Error removing learned bgp route [ %s ]", err)
					}
				} else {
					monitorUpdate.IsWithdraw = false
					b.learnedRoutes = append(b.learnedRoutes, *monitorUpdate)
					log.Debugf("Learned routes: %v ", monitorUpdate)

					err = addNetlinkRoute(monitorUpdate.BgpPrefix, monitorUpdate.NextHop, b.ethIface)
					if err != nil {
						log.Debugf("Add route results [ %s ]", err)
					}

					log.Infof("Updated the local prefix cache from the newly learned BGP update:")
					for n, entry := range b.learnedRoutes {
						log.Debugf("%d - %+v", n+1, entry)
					}
				}
			}
			log.Debugf("Verbose update details: %s", monitorUpdate)
		case neighbors := <-events:
			for _, neighbor := range neighbors {
				neighboraddr := strings.Split(neighbor.Key, "/")[1]
				_, ok := b.neighborlist[neighboraddr]
				if neighboraddr != b.server.String() && !ok {
					b.neighborlist[neighboraddr] = string(neighbor.Value)
					neighborAS, _ := strconv.ParseUint(string(neighbor.Value), 10, 32)
					log.Debugf("BGP neighbor add %s", neighboraddr)
					peer := &api.Peer{
						// TODO: Debug Unresolved in Gobgp API master
						// NighborAddress: neig