// Create creates a bridge device and returns the interface.
// If the device already exists, returns the existing interface.
func (Bridge) Create(name string, ip net.IP, subnet *net.IPNet) (intf *net.Interface, err error) {
	netlinkMu.Lock()
	defer netlinkMu.Unlock()

	if intf, _ := net.InterfaceByName(name); intf != nil {
		return intf, nil
	}

	link := &netlink.Bridge{LinkAttrs: netlink.LinkAttrs{Name: name}}

	if err := netlink.LinkAdd(link); err != nil && err.Error() != "file exists" {
		return nil, fmt.Errorf("devices: create bridge: %v", err)
	}

	hAddr, _ := net.ParseMAC(randMacAddr())
	err = netlink.LinkSetHardwareAddr(link, hAddr)
	if err != nil {
		return nil, fmt.Errorf("devices: set hardware address: %v", err)
	}

	if intf, err = net.InterfaceByName(name); err != nil {
		return nil, fmt.Errorf("devices: look up created bridge interface: %v", err)
	}

	addr := &netlink.Addr{IPNet: &net.IPNet{IP: ip, Mask: subnet.Mask}}
	if err = netlink.AddrAdd(link, addr); err != nil && err.Error() != "file exists" {
		return nil, fmt.Errorf("devices: add IP to bridge: %v", err)
	}
	return intf, nil
}

func SetHWAddrByIP(ifName string, ip4 net.IP, ip6 net.IP) error {
	iface, err := netlink.LinkByName(ifName)
	if err != nil {
		return fmt.Errorf("failed to lookup %q: %v", ifName, err)
	}

	switch {
	case ip4 == nil && ip6 == nil:
		return fmt.Errorf("neither ip4 or ip6 specified")

	case ip4 != nil:
		{
			hwAddr, err := hwaddr.GenerateHardwareAddr4(ip4, hwaddr.PrivateMACPrefix)
			if err != nil {
				return fmt.Errorf("failed to generate hardware addr: %v", err)
			}
			if err = netlink.LinkSetHardwareAddr(iface, hwAddr); err != nil {
				return fmt.Errorf("failed to add hardware addr to %q: %v", ifName, err)
			}
		}
	case ip6 != nil:
		// TODO: IPv6
	}

	return nil
}

// SetInterfaceMac : Set mac address of an interface
func SetInterfaceMac(name string, macaddr string) error {
	iface, err := netlink.LinkByName(name)
	if err != nil {
		return err
	}
	hwaddr, err := net.ParseMAC(macaddr)
	if err != nil {
		return err
	}
	return netlink.LinkSetHardwareAddr(iface, hwaddr)
}

func SetVethMac(vethNameHost, mac string) error {
	addr, err := net.ParseMAC(mac)
	if err != nil {
		return errors.Wrap(err, "Veth setting error")
	}
	return netlink.LinkSetHardwareAddr(&netlink.Veth{
		LinkAttrs: netlink.LinkAttrs{
			Name: vethNameHost,
		},
	}, addr)
}

// Wire up a tap interface for communicating with the guest. Returns the name
// of the created tap interface.
func wireTapInterface(config *netConfig) string {
	// Drop link on eth0 before configuring anything
	eth0, err := netlink.LinkByName("eth0")
	if err != nil {
		log.Fatalf("LinkByName(eth0): %v", err)
	}
	if err := netlink.LinkSetDown(eth0); err != nil {
		log.Fatalf("LinkSetDown(eth0): %v", err)
	}
	// Flush any L3 addresses on eth0
	if err := flushAddresses(eth0); err != nil {
		log.Fatalf("flushAddresses(eth0): %v", err)
	}
	// Generate and set random MAC address for eth0
	eth0Addr := generateHardwareAddr()
	if err := netlink.LinkSetHardwareAddr(eth0, eth0Addr); err != nil {
		log.Fatalf("LinkSetHardwareAddr(eth0): %v", err)
	}
	// Create "tap0" (interface to guest)
	tap0Attrs := netlink.NewLinkAttrs()
	tap0Attrs.Name = "tap0"
	tap0 := &netlink.Tuntap{tap0Attrs, netlink.TUNTAP_MODE_TAP}
	if err := netlink.LinkAdd(tap0); err != nil {
		log.Fatalf("LinkAdd(tap0): %v", err)
	}
	// Create a new bridge, br0 and add eth0 and tap0 to it
	br0Attrs := netlink.NewLinkAttrs()
	br0Attrs.Name = "br0"
	br0 := &netlink.Bridge{br0Attrs}
	if err := netlink.LinkAdd(br0); err != nil {
		log.Fatalf("LinkAdd(br0): %v", err)
	}
	if err := netlink.LinkSetMaster(eth0, br0); err != nil {
		log.Fatalf("LinkSetMaster(eth0, br0): %v", err)
	}
	if err := netlink.LinkSetMaster(tap0, br0); err != nil {
		log.Fatalf("LinkSetMaster(tap0, br0): %v", err)
	}
	// Set all links up
	if err := netlink.LinkSetUp(tap0); err != nil {
		log.Fatalf("LinkSetUp(tap0): %v", err)
	}
	if err := netlink.LinkSetUp(eth0); err != nil {
		log.Fatalf("LinkSetUp(eth0): %v", err)
	}
	if err := netlink.LinkSetUp(br0); err != nil {
		log.Fatalf("LinkSetUp(br0): %v", err)
	}
	return tap0Attrs.Name
}

// SetupDevice create a new bridge interface/
func setupDevice(config *networkConfiguration, i *bridgeInterface) error {
	var setMac bool

	// We only attempt to create the bridge when the requested device name is
	// the default one.
	if config.BridgeName != DefaultBridgeName && !config.AllowNonDefaultBridge {
		return NonDefaultBridgeExistError(config.BridgeName)
	}

	// Set the bridgeInterface netlink.Bridge.
	i.Link = &netlink.Bridge{
		LinkAttrs: netlink.LinkAttrs{
			Name: config.BridgeName,
		},
	}

	// Only set the bridge's MAC address if the kernel version is > 3.3, as it
	// was not supported before that.
	kv, err := kernel.GetKernelVersion()
	if err != nil {
		logrus.Errorf("Failed to check kernel versions: %v. Will not assign a MAC address to the bridge interface", err)
	} else {
		setMac = kv.Kernel > 3 || (kv.Kernel == 3 && kv.Major >= 3)
	}

	if err = netlink.LinkAdd(i.Link); err != nil {
		logrus.Debugf("Failed to create bridge %s via netlink. Trying ioctl", config.BridgeName)
		return ioctlCreateBridge(config.BridgeName, setMac)
	}

	if setMac {
		hwAddr := netutils.GenerateRandomMAC()
		if err = netlink.LinkSetHardwareAddr(i.Link, hwAddr); err != nil {
			return fmt.Errorf("failed to set bridge mac-address %s : %s", hwAddr, err.Error())
		}
		logrus.Debugf("Setting bridge mac address to %s", hwAddr)
		logrus.Debugf("call recieved to bridge driver, PLUMgrid")
	}
	return err
}

// Join method is invoked when a Sandbox is attached to an endpoint.
func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, options map[string]interface{}) error {
	if err := validateID(nid, eid); err != nil {
		return err
	}

	n := d.network(nid)
	if n == nil {
		return fmt.Errorf("could not find network with id %s", nid)
	}

	ep := n.endpoint(eid)
	if ep == nil {
		return fmt.Errorf("could not find endpoint with id %s", eid)
	}

	s := n.getSubnetforIP(ep.addr)
	if s == nil {
		return fmt.Errorf("could not find subnet for endpoint %s", eid)
	}

	if err := n.obtainVxlanID(s); err != nil {
		return fmt.Errorf("couldn't get vxlan id for %q: %v", s.subnetIP.String(), err)
	}

	if err := n.joinSandbox(); err != nil {
		return fmt.Errorf("network sandbox join failed: %v", err)
	}

	if err := n.joinSubnetSandbox(s); err != nil {
		return fmt.Errorf("subnet sandbox join failed for %q: %v", s.subnetIP.String(), err)
	}

	// joinSubnetSandbox gets called when an endpoint comes up on a new subnet in the
	// overlay network. Hence the Endpoint count should be updated outside joinSubnetSandbox
	n.incEndpointCount()

	sbox := n.sandbox()

	name1, name2, err := createVethPair()
	if err != nil {
		return err
	}

	// Set the container interface and its peer MTU to 1450 to allow
	// for 50 bytes vxlan encap (inner eth header(14) + outer IP(20) +
	// outer UDP(8) + vxlan header(8))
	veth, err := netlink.LinkByName(name1)
	if err != nil {
		return fmt.Errorf("cound not find link by name %s: %v", name1, err)
	}
	err = netlink.LinkSetMTU(veth, vxlanVethMTU)
	if err != nil {
		return err
	}

	if err := sbox.AddInterface(name1, "veth",
		sbox.InterfaceOptions().Master(s.brName)); err != nil {
		return fmt.Errorf("could not add veth pair inside the network sandbox: %v", err)
	}

	veth, err = netlink.LinkByName(name2)
	if err != nil {
		return fmt.Errorf("could not find link by name %s: %v", name2, err)
	}
	err = netlink.LinkSetMTU(veth, vxlanVethMTU)
	if err != nil {
		return err
	}

	if err := netlink.LinkSetHardwareAddr(veth, ep.mac); err != nil {
		return fmt.Errorf("could not set mac address (%v) to the container interface: %v", ep.mac, err)
	}

	for _, sub := range n.subnets {
		if sub == s {
			continue
		}
		if err := jinfo.AddStaticRoute(sub.subnetIP, types.NEXTHOP, s.gwIP.IP); err != nil {
			log.Errorf("Adding subnet %s static route in network %q failed\n", s.subnetIP, n.id)
		}
	}

	if iNames := jinfo.InterfaceName(); iNames != nil {
		err = iNames.SetNames(name2, "eth")
		if err != nil {
			return err
		}
	}

	d.peerDbAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac,
		net.ParseIP(d.bindAddress), true)
	d.pushLocalEndpointEvent("join", nid, eid)

	return nil
}

// Join method is invoked when a Sandbox is attached to an endpoint.
func (d *driver) Join(nid, eid types.UUID, sboxKey string, jinfo driverapi.JoinInfo, options map[string]interface{}) error {
	if err := validateID(nid, eid); err != nil {
		return err
	}

	n := d.network(nid)
	if n == nil {
		return fmt.Errorf("could not find network with id %s", nid)
	}

	ep := n.endpoint(eid)
	if ep == nil {
		return fmt.Errorf("could not find endpoint with id %s", eid)
	}

	if err := n.joinSandbox(); err != nil {
		return fmt.Errorf("network sandbox join failed: %v",
			err)
	}

	sbox := n.sandbox()

	name1, name2, err := createVethPair()
	if err != nil {
		return err
	}

	if err := sbox.AddInterface(name1, "veth",
		sbox.InterfaceOptions().Master("bridge1")); err != nil {
		return fmt.Errorf("could not add veth pair inside the network sandbox: %v", err)
	}

	veth, err := netlink.LinkByName(name2)
	if err != nil {
		return fmt.Errorf("could not find link by name %s: %v", name2, err)
	}

	if err := netlink.LinkSetHardwareAddr(veth, ep.mac); err != nil {
		return fmt.Errorf("could not set mac address to the container interface: %v", err)
	}

	for _, iNames := range jinfo.InterfaceNames() {
		// Make sure to set names on the correct interface ID.
		if iNames.ID() == 1 {
			err = iNames.SetNames(name2, "eth")
			if err != nil {
				return err
			}
		}
	}

	err = jinfo.SetGateway(bridgeIP.IP)
	if err != nil {
		return err
	}

	d.peerDbAdd(nid, eid, ep.addr.IP, ep.mac,
		d.serfInstance.LocalMember().Addr, true)
	d.notifyCh <- ovNotify{
		action: "join",
		nid:    nid,
		eid:    eid,
	}

	return nil
}

//.........这里部分代码省略.........
	}
	defer func() {
		if err != nil {
			netlink.LinkDel(sbox)
		}
	}()

	// Add bridge inherited attributes to pipe interfaces
	if config.Mtu != 0 {
		err = netlink.LinkSetMTU(host, config.Mtu)
		if err != nil {
			return err
		}
		err = netlink.LinkSetMTU(sbox, config.Mtu)
		if err != nil {
			return err
		}
	}

	// Attach host side pipe interface into the bridge
	if err = netlink.LinkSetMaster(host,
		&netlink.Bridge{LinkAttrs: netlink.LinkAttrs{Name: config.BridgeName}}); err != nil {
		return err
	}

	if !config.EnableUserlandProxy {
		err = netlink.LinkSetHairpin(host, true)
		if err != nil {
			return err
		}
	}

	// v4 address for the sandbox side pipe interface
	ip4, err := ipAllocator.RequestIP(n.bridge.bridgeIPv4, nil)
	if err != nil {
		return err
	}
	ipv4Addr := &net.IPNet{IP: ip4, Mask: n.bridge.bridgeIPv4.Mask}

	// Set the sbox's MAC. If specified, use the one configured by user, otherwise generate one based on IP.
	mac := electMacAddress(epConfig, ip4)
	err = netlink.LinkSetHardwareAddr(sbox, mac)
	if err != nil {
		return err
	}
	endpoint.macAddress = mac

	// v6 address for the sandbox side pipe interface
	ipv6Addr = &net.IPNet{}
	if config.EnableIPv6 {
		var ip6 net.IP

		network := n.bridge.bridgeIPv6
		if config.FixedCIDRv6 != nil {
			network = config.FixedCIDRv6
		}

		ones, _ := network.Mask.Size()
		if ones <= 80 {
			ip6 = make(net.IP, len(network.IP))
			copy(ip6, network.IP)
			for i, h := range mac {
				ip6[i+10] = h
			}
		}

		ip6, err := ipAllocator.RequestIP(network, ip6)
		if err != nil {
			return err
		}

		ipv6Addr = &net.IPNet{IP: ip6, Mask: network.Mask}
	}

	// Create the sandbox side pipe interface
	intf := &sandbox.Interface{}
	intf.SrcName = name2
	intf.DstName = containerVethPrefix
	intf.Address = ipv4Addr

	if config.EnableIPv6 {
		intf.AddressIPv6 = ipv6Addr
	}

	// Store the interface in endpoint, this is needed for cleanup on DeleteEndpoint()
	endpoint.intf = intf

	err = epInfo.AddInterface(ifaceID, endpoint.macAddress, *ipv4Addr, *ipv6Addr)
	if err != nil {
		return err
	}

	// Program any required port mapping and store them in the endpoint
	endpoint.portMapping, err = allocatePorts(epConfig, intf, config.DefaultBindingIP, config.EnableUserlandProxy)
	if err != nil {
		return err
	}

	return nil
}

//.........这里部分代码省略.........
	// Get the sandbox side pipe interface handler
	sbox, err := netlink.LinkByName(containerIfName)
	if err != nil {
		return types.InternalErrorf("failed to find sandbox side interface %s: %v", containerIfName, err)
	}
	defer func() {
		if err != nil {
			netlink.LinkDel(sbox)
		}
	}()

	n.Lock()
	config := n.config
	n.Unlock()

	// Add bridge inherited attributes to pipe interfaces
	if config.Mtu != 0 {
		err = netlink.LinkSetMTU(host, config.Mtu)
		if err != nil {
			return types.InternalErrorf("failed to set MTU on host interface %s: %v", hostIfName, err)
		}
		err = netlink.LinkSetMTU(sbox, config.Mtu)
		if err != nil {
			return types.InternalErrorf("failed to set MTU on sandbox interface %s: %v", containerIfName, err)
		}
	}

	// Attach host side pipe interface into the bridge
	if err = addToBridge(hostIfName, config.BridgeName); err != nil {
		return fmt.Errorf("adding interface %s to bridge %s failed: %v", hostIfName, config.BridgeName, err)
	}

	if !dconfig.EnableUserlandProxy {
		err = setHairpinMode(host, true)
		if err != nil {
			return err
		}
	}

	// Create the sandbox side pipe interface
	endpoint.srcName = containerIfName
	endpoint.macAddress = ifInfo.MacAddress()
	endpoint.addr = ifInfo.Address()
	endpoint.addrv6 = ifInfo.AddressIPv6()

	// Down the interface before configuring mac address.
	if err = netlink.LinkSetDown(sbox); err != nil {
		return fmt.Errorf("could not set link down for container interface %s: %v", containerIfName, err)
	}

	// Set the sbox's MAC. If specified, use the one configured by user, otherwise generate one based on IP.
	if endpoint.macAddress == nil {
		endpoint.macAddress = electMacAddress(epConfig, endpoint.addr.IP)
		if err := ifInfo.SetMacAddress(endpoint.macAddress); err != nil {
			return err
		}
	}
	err = netlink.LinkSetHardwareAddr(sbox, endpoint.macAddress)
	if err != nil {
		return fmt.Errorf("could not set mac address for container interface %s: %v", containerIfName, err)
	}

	// Up the host interface after finishing all netlink configuration
	if err = netlink.LinkSetUp(host); err != nil {
		return fmt.Errorf("could not set link up for host interface %s: %v", hostIfName, err)
	}

	if endpoint.addrv6 == nil && config.EnableIPv6 {
		var ip6 net.IP
		network := n.bridge.bridgeIPv6
		if config.AddressIPv6 != nil {
			network = config.AddressIPv6
		}

		ones, _ := network.Mask.Size()
		if ones > 80 {
			err = types.ForbiddenErrorf("Cannot self generate an IPv6 address on network %v: At least 48 host bits are needed.", network)
			return err
		}

		ip6 = make(net.IP, len(network.IP))
		copy(ip6, network.IP)
		for i, h := range endpoint.macAddress {
			ip6[i+10] = h
		}

		endpoint.addrv6 = &net.IPNet{IP: ip6, Mask: network.Mask}
		if err := ifInfo.SetIPAddress(endpoint.addrv6); err != nil {
			return err
		}
	}

	// Program any required port mapping and store them in the endpoint
	endpoint.portMapping, err = n.allocatePorts(epConfig, endpoint, config.DefaultBindingIP, d.config.EnableUserlandProxy)
	if err != nil {
		return err
	}

	return nil
}

//.........这里部分代码省略.........
			if err != nil {
				log.Errorf("Error converting GBP to bool: %v", err)
				return nil, err
			}
			vxlan.GBP = GBP
		}
		if k == "Age" {
			Age, err := strconv.Atoi(v)
			if err != nil {
				log.Errorf("Error converting Age to int: %v", err)
				return nil, err
			}
			vxlan.Age = Age
		}
		if k == "Limit" {
			Limit, err := strconv.Atoi(v)
			if err != nil {
				log.Errorf("Error converting Limit to int: %v", err)
				return nil, err
			}
			vxlan.Limit = Limit
		}
		if k == "Port" {
			Port, err := strconv.Atoi(v)
			if err != nil {
				log.Errorf("Error converting Port to int: %v", err)
				return nil, err
			}
			vxlan.Port = Port
		}
		if k == "PortLow" {
			PortLow, err := strconv.Atoi(v)
			if err != nil {
				log.Errorf("Error converting PortLow to int: %v", err)
				return nil, err
			}
			vxlan.PortLow = PortLow
		}
		if k == "PortHigh" {
			PortHigh, err := strconv.Atoi(v)
			if err != nil {
				log.Errorf("Error converting PortHigh to int: %v", err)
				return nil, err
			}
			vxlan.PortHigh = PortHigh
		}
	}

	if d.vtepdev != "" {
		vtepDev, err := netlink.LinkByName(d.vtepdev)
		if err != nil {
			log.Errorf("Error getting vtepdev interface: %v", err)
			return nil, err
		}
		vxlan.VtepDevIndex = vtepDev.Attrs().Index
	}

	err := netlink.LinkAdd(vxlan)
	if err != nil {
		log.Errorf("Error adding vxlan interface: %v", err)
		return nil, err
	}

	// Parse interface options
	for k, v := range net.Options {
		if k == "vxlanHardwareAddr" {
			hardwareAddr, err := gonet.ParseMAC(v)
			if err != nil {
				log.Errorf("Error parsing mac address: %v", err)
				return nil, err
			}
			err = netlink.LinkSetHardwareAddr(vxlan, hardwareAddr)
			if err != nil {
				log.Errorf("Error setting mac address: %v", err)
				return nil, err
			}
		}
		if k == "vxlanMTU" {
			mtu, err := strconv.Atoi(v)
			if err != nil {
				log.Errorf("Error converting MTU to int: %v", err)
				return nil, err
			}
			err = netlink.LinkSetMTU(vxlan, mtu)
			if err != nil {
				log.Errorf("Error setting MTU: %v", err)
				return nil, err
			}
		}
	}

	// bring interfaces up
	err = netlink.LinkSetUp(vxlan)
	if err != nil {
		log.Errorf("Error bringing up vxlan: %v", err)
		return nil, err
	}

	return vxlan, nil
}

func setInterfaceMAC(iface netlink.Link, i *nwIface) error {
	if i.MacAddress() == nil {
		return nil
	}
	return netlink.LinkSetHardwareAddr(iface, i.MacAddress())
}

func (v *veth) initialize(config *network) error {
	peer := config.TempVethPeerName
	if peer == "" {
		return fmt.Errorf("peer is not specified")
	}
	child, err := netlink.LinkByName(peer)
	if err != nil {
		return err
	}
	if err := netlink.LinkSetDown(child); err != nil {
		return err
	}
	if err := netlink.LinkSetName(child, config.Name); err != nil {
		return err
	}
	// get the interface again after we changed the name as the index also changes.
	if child, err = netlink.LinkByName(config.Name); err != nil {
		return err
	}
	if config.MacAddress != "" {
		mac, err := net.ParseMAC(config.MacAddress)
		if err != nil {
			return err
		}
		if err := netlink.LinkSetHardwareAddr(child, mac); err != nil {
			return err
		}
	}
	ip, err := netlink.ParseAddr(config.Address)
	if err != nil {
		return err
	}
	if err := netlink.AddrAdd(child, ip); err != nil {
		return err
	}
	if config.IPv6Address != "" {
		ip6, err := netlink.ParseAddr(config.IPv6Address)
		if err != nil {
			return err
		}
		if err := netlink.AddrAdd(child, ip6); err != nil {
			return err
		}
	}
	if err := netlink.LinkSetMTU(child, config.Mtu); err != nil {
		return err
	}
	if err := netlink.LinkSetUp(child); err != nil {
		return err
	}
	if config.Gateway != "" {
		gw := net.ParseIP(config.Gateway)
		if err := netlink.RouteAdd(&netlink.Route{
			Scope:     netlink.SCOPE_UNIVERSE,
			LinkIndex: child.Attrs().Index,
			Gw:        gw,
		}); err != nil {
			return err
		}
	}
	if config.IPv6Gateway != "" {
		gw := net.ParseIP(config.IPv6Gateway)
		if err := netlink.RouteAdd(&netlink.Route{
			Scope:     netlink.SCOPE_UNIVERSE,
			LinkIndex: child.Attrs().Index,
			Gw:        gw,
		}); err != nil {
			return err
		}
	}
	return nil
}

// setup sets up networking through CNI using the given ns/name and sandbox ID.
// TODO: Don't pass the pod to this method, it only needs it for bandwidth
// shaping and hostport management.
func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kubecontainer.ContainerID, pod *v1.Pod) error {
	// Bring up container loopback interface
	if _, err := plugin.addContainerToNetwork(plugin.loConfig, "lo", namespace, name, id); err != nil {
		return err
	}

	// Hook container up with our bridge
	res, err := plugin.addContainerToNetwork(plugin.netConfig, network.DefaultInterfaceName, namespace, name, id)
	if err != nil {
		return err
	}
	if res.IP4 == nil {
		return fmt.Errorf("CNI plugin reported no IPv4 address for container %v.", id)
	}
	ip4 := res.IP4.IP.IP.To4()
	if ip4 == nil {
		return fmt.Errorf("CNI plugin reported an invalid IPv4 address for container %v: %+v.", id, res.IP4)
	}

	// Explicitly assign mac address to cbr0. If bridge mac address is not explicitly set will adopt the lowest MAC address of the attached veths.
	// TODO: Remove this once upstream cni bridge plugin handles this
	link, err := netlink.LinkByName(BridgeName)
	if err != nil {
		return fmt.Errorf("failed to lookup %q: %v", BridgeName, err)
	}
	macAddr, err := generateHardwareAddr(plugin.gateway)
	if err != nil {
		return err
	}
	glog.V(3).Infof("Configure %q mac address to %v", BridgeName, macAddr)
	err = netlink.LinkSetHardwareAddr(link, macAddr)
	if err != nil {
		return fmt.Errorf("Failed to configure %q mac address to %q: %v", BridgeName, macAddr, err)
	}

	// Put the container bridge into promiscuous mode to force it to accept hairpin packets.
	// TODO: Remove this once the kernel bug (#20096) is fixed.
	// TODO: check and set promiscuous mode with netlink once vishvananda/netlink supports it
	if plugin.hairpinMode == componentconfig.PromiscuousBridge {
		output, err := plugin.execer.Command("ip", "link", "show", "dev", BridgeName).CombinedOutput()
		if err != nil || strings.Index(string(output), "PROMISC") < 0 {
			_, err := plugin.execer.Command("ip", "link", "set", BridgeName, "promisc", "on").CombinedOutput()
			if err != nil {
				return fmt.Errorf("Error setting promiscuous mode on %s: %v", BridgeName, err)
			}
		}
		// configure the ebtables rules to eliminate duplicate packets by best effort
		plugin.syncEbtablesDedupRules(macAddr)
	}

	plugin.podIPs[id] = ip4.String()

	// The host can choose to not support "legacy" features. The remote
	// shim doesn't support it (#35457), but the kubelet does.
	if !plugin.host.SupportsLegacyFeatures() {
		return nil
	}

	// The first SetUpPod call creates the bridge; get a shaper for the sake of
	// initialization
	shaper := plugin.shaper()

	ingress, egress, err := bandwidth.ExtractPodBandwidthResources(pod.Annotations)
	if err != nil {
		return fmt.Errorf("Error reading pod bandwidth annotations: %v", err)
	}
	if egress != nil || ingress != nil {
		if err := shaper.ReconcileCIDR(fmt.Sprintf("%s/32", ip4.String()), egress, ingress); err != nil {
			return fmt.Errorf("Failed to add pod to shaper: %v", err)
		}
	}

	// Open any hostports the pod's containers want
	activePods, err := plugin.getActivePods()
	if err != nil {
		return err
	}

	newPod := &hostport.ActivePod{Pod: pod, IP: ip4}
	if err := plugin.hostportHandler.OpenPodHostportsAndSync(newPod, BridgeName, activePods); err != nil {
		return err
	}

	return nil
}

// Join method is invoked when a Sandbox is attached to an endpoint.
func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, options map[string]interface{}) error {
	if err := validateID(nid, eid); err != nil {
		return err
	}

	n := d.network(nid)
	if n == nil {
		return fmt.Errorf("could not find network with id %s", nid)
	}

	ep := n.endpoint(eid)
	if ep == nil {
		return fmt.Errorf("could not find endpoint with id %s", eid)
	}

	if err := n.joinSandbox(); err != nil {
		return fmt.Errorf("network sandbox join failed: %v",
			err)
	}

	sbox := n.sandbox()

	name1, name2, err := createVethPair()
	if err != nil {
		return err
	}

	// Set the container interface and its peer MTU to 1450 to allow
	// for 50 bytes vxlan encap (inner eth header(14) + outer IP(20) +
	// outer UDP(8) + vxlan header(8))
	veth, err := netlink.LinkByName(name1)
	if err != nil {
		return fmt.Errorf("cound not find link by name %s: %v", name1, err)
	}
	err = netlink.LinkSetMTU(veth, vxlanVethMTU)
	if err != nil {
		return err
	}

	if err := sbox.AddInterface(name1, "veth",
		sbox.InterfaceOptions().Master("bridge1")); err != nil {
		return fmt.Errorf("could not add veth pair inside the network sandbox: %v", err)
	}

	veth, err = netlink.LinkByName(name2)
	if err != nil {
		return fmt.Errorf("could not find link by name %s: %v", name2, err)
	}
	err = netlink.LinkSetMTU(veth, vxlanVethMTU)
	if err != nil {
		return err
	}

	if err := netlink.LinkSetHardwareAddr(veth, ep.mac); err != nil {
		return fmt.Errorf("could not set mac address to the container interface: %v", err)
	}

	if iNames := jinfo.InterfaceName(); iNames != nil {
		err = iNames.SetNames(name2, "eth")
		if err != nil {
			return err
		}
	}

	err = jinfo.SetGateway(bridgeIP.IP)
	if err != nil {
		return err
	}

	d.peerDbAdd(nid, eid, ep.addr.IP, ep.mac,
		d.serfInstance.LocalMember().Addr, true)
	d.notifyCh <- ovNotify{
		action: "join",
		nid:    nid,
		eid:    eid,
	}

	return nil
}

//.........这里部分代码省略.........
	config := n.config
	n.Unlock()

	// Add bridge inherited attributes to pipe interfaces
	if config.Mtu != 0 {
		err = netlink.LinkSetMTU(host, config.Mtu)
		if err != nil {
			return err
		}
		err = netlink.LinkSetMTU(sbox, config.Mtu)
		if err != nil {
			return err
		}
	}

	// Attach host side pipe interface into the bridge
	if err = addToBridge(hostIfName, config.BridgeName); err != nil {
		return fmt.Errorf("adding interface %s to bridge %s failed: %v", hostIfName, config.BridgeName, err)
	}

	if !config.EnableUserlandProxy {
		err = setHairpinMode(host, true)
		if err != nil {
			return err
		}
	}

	// v4 address for the sandbox side pipe interface
	ip4, err := ipAllocator.RequestIP(n.bridge.bridgeIPv4, nil)
	if err != nil {
		return err
	}
	ipv4Addr := &net.IPNet{IP: ip4, Mask: n.bridge.bridgeIPv4.Mask}

	// Down the interface before configuring mac address.
	if err = netlink.LinkSetDown(sbox); err != nil {
		return fmt.Errorf("could not set link down for container interface %s: %v", containerIfName, err)
	}

	// Set the sbox's MAC. If specified, use the one configured by user, otherwise generate one based on IP.
	mac := electMacAddress(epConfig, ip4)
	err = netlink.LinkSetHardwareAddr(sbox, mac)
	if err != nil {
		return fmt.Errorf("could not set mac address for container interface %s: %v", containerIfName, err)
	}
	endpoint.macAddress = mac

	// Up the host interface after finishing all netlink configuration
	if err = netlink.LinkSetUp(host); err != nil {
		return fmt.Errorf("could not set link up for host interface %s: %v", hostIfName, err)
	}

	// v6 address for the sandbox side pipe interface
	ipv6Addr = &net.IPNet{}
	if config.EnableIPv6 {
		var ip6 net.IP

		network := n.bridge.bridgeIPv6
		if config.FixedCIDRv6 != nil {
			network = config.FixedCIDRv6
		}

		ones, _ := network.Mask.Size()
		if ones <= 80 {
			ip6 = make(net.IP, len(network.IP))
			copy(ip6, network.IP)
			for i, h := range mac {
				ip6[i+10] = h
			}
		}

		ip6, err := ipAllocator.RequestIP(network, ip6)
		if err != nil {
			return err
		}

		ipv6Addr = &net.IPNet{IP: ip6, Mask: network.Mask}
	}

	// Create the sandbox side pipe interface
	endpoint.srcName = containerIfName
	endpoint.addr = ipv4Addr

	if config.EnableIPv6 {
		endpoint.addrv6 = ipv6Addr
	}

	err = epInfo.AddInterface(ifaceID, endpoint.macAddress, *ipv4Addr, *ipv6Addr)
	if err != nil {
		return err
	}

	// Program any required port mapping and store them in the endpoint
	endpoint.portMapping, err = n.allocatePorts(epConfig, endpoint, config.DefaultBindingIP, config.EnableUserlandProxy)
	if err != nil {
		return err
	}

	return nil
}

func (e *endpoints) create(eid string, ifInfo *driverapi.EndpointInterface, niConfig networkConfig) (err error) {
	ep := endpoint{}

	// Generate a name for what will be the host side pipe interface
	hostIfName, err := netutils.GenerateIfaceName(vethPrefix, vethLen)
	if err != nil {
		return err
	}

	// Generate a name for what will be the sandbox side pipe interface
	containerIfName, err := netutils.GenerateIfaceName(vethPrefix, vethLen)
	if err != nil {
		return err
	}

	// Generate and add the interface pipe host <-> sandbox
	veth := &netlink.Veth{
		LinkAttrs: netlink.LinkAttrs{Name: hostIfName, TxQLen: 0},
		PeerName:  containerIfName,
	}
	if err = netlink.LinkAdd(veth); err != nil {
		return types.InternalErrorf("failed to add the host (%s) <=> sandbox (%s) pair interfaces: %v", hostIfName, containerIfName, err)
	}

	// Get the host side pipe interface handler
	host, err := netlink.LinkByName(hostIfName)
	if err != nil {
		return types.InternalErrorf("failed to find host side interface %s: %v", hostIfName, err)
	}
	defer func() {
		if err != nil {
			netlink.LinkDel(host)
		}
	}()

	// Get the sandbox side pipe interface handler
	sbox, err := netlink.LinkByName(containerIfName)
	if err != nil {
		return types.InternalErrorf("failed to find sandbox side interface %s: %v", containerIfName, err)
	}
	defer func() {
		if err != nil {
			netlink.LinkDel(sbox)
		}
	}()

	// Add bridge inherited attributes to pipe interfaces
	if niConfig.Mtu != 0 {
		err = netlink.LinkSetMTU(host, niConfig.Mtu)
		if err != nil {
			return types.InternalErrorf("failed to set MTU on host interface %s: %v", hostIfName, err)
		}
		err = netlink.LinkSetMTU(sbox, niConfig.Mtu)
		if err != nil {
			return types.InternalErrorf("failed to set MTU on sandbox interface %s: %v", containerIfName, err)
		}
	}

	// Attach host side pipe interface into the bridge
	br, err := netlink.LinkByName(niConfig.BridgeName)
	if err != nil {
		return types.InternalErrorf("failed to find bridge by name %s: %v", niConfig.BridgeName, err)
	}
	if err = netlink.LinkSetMaster(host, br.(*netlink.Bridge)); err != nil {
		return fmt.Errorf("adding interface %s to bridge %s failed: %v", hostIfName, niConfig.BridgeName, err)
	}

	// Create the sandbox side pipe interface
	ep.ifname = containerIfName
	ep.addr, _, err = net.ParseCIDR(ifInfo.Address)
	if err != nil {
		return fmt.Errorf("ipv4 adress unparseable")
	}
	/*
		_, ep.addrv6, err = net.ParseCIDR(ifInfo.AddressIPv6)
		if err != nil {
			return fmt.Errorf("ipv6 adress unparseable")
		}
	*/

	if ifInfo.MacAddress != "" {
		ep.mac, err = net.ParseMAC(ifInfo.MacAddress)
		if err != nil {
			return fmt.Errorf("mac adress unparseable")
		}
		// Down the interface before configuring mac address.
		if err = netlink.LinkSetDown(sbox); err != nil {
			return fmt.Errorf("could not set link down for container interface %s: %v", containerIfName, err)
		}

		err = netlink.LinkSetHardwareAddr(sbox, ep.mac)
		if err != nil {
			return fmt.Errorf("could not set mac address for container interface %s: %v", containerIfName, err)
		}

		if err = netlink.LinkSetUp(sbox); err != nil {
			return fmt.Errorf("could not set link up for container interface %s: %v", containerIfName, err)
		}
	} else {
		// Get existing mac address from interface
//.........这里部分代码省略.........