func resourceLXCBridgeDelete(d *schema.ResourceData, meta interface{}) error {
	bridgeIndex, err := strconv.Atoi(d.Id())
	if err != nil {
		return fmt.Errorf("Internal error reading resource ID: %v", err)
	}

	bridge, err := netlink.LinkByIndex(bridgeIndex)
	if err != nil {
		return fmt.Errorf("Unable to find bridge %v: %v", bridgeIndex, err)
	}

	links, err := netlink.LinkList()
	if err != nil {
		return fmt.Errorf("Error listing interfaces: %v", err)
	}

	bridgeEmpty := true
	for _, link := range links {
		if link.Attrs().MasterIndex == bridge.Attrs().Index {
			bridgeEmpty = false
			log.Printf("[INFO] Link %s is still attached to bridge %s", link.Attrs().Name, bridge.Attrs().Name)
		}
	}

	if bridgeEmpty == false {
		return fmt.Errorf("Unable to delete bridge %s. Interfaces are still attached to it.", bridge.Attrs().Name)
	} else {
		if err := netlink.LinkDel(bridge); err != nil {
			return fmt.Errorf("Error deleting bridge: %s", err)
		}
	}

	return nil
}

func testAccCheckLXCBridgeExists(t *testing.T, n string, bridge *netlink.Link) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %v", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ID is set")
		}

		bridgeIndex, err := strconv.Atoi(rs.Primary.ID)
		if err != nil {
			return fmt.Errorf("Internal error reading resource ID.")
		}
		br, err := netlink.LinkByIndex(bridgeIndex)
		if err != nil {
			return fmt.Errorf("Error searching for bridge.")
		} else {
			*bridge = br
			return nil
		}

		return fmt.Errorf("Unable to find bridge.")
	}
}

// gatherLinks listens for netlink notifications and collects the list of links
// created during the subscription window.
func gatherLinks(ch <-chan netlink.LinkUpdate) ([]netlink.Link, error) {
	linkSet := make(map[int32]bool)
	timeout := time.After(1000 * time.Millisecond)
	Debug.Printf("Waiting for link updates\n")
outer:
	for {
		select {
		case update := <-ch:
			if update.Header.Type == syscall.RTM_NEWLINK {
				if _, ok := update.Link.(*netlink.Veth); ok {
					linkSet[update.Index] = true
				}
			} else if update.Header.Type == syscall.RTM_DELLINK {
				delete(linkSet, update.Index)
			}
		case <-timeout:
			break outer
		}
	}
	links := []netlink.Link{}
	for index, _ := range linkSet {
		l, err := netlink.LinkByIndex(int(index))
		if err == nil {
			links = append(links, l)
		}

	}
	return links, nil
}

func ensureLink(vxlan *netlink.Vxlan) (*netlink.Vxlan, error) {
	err := netlink.LinkAdd(vxlan)
	if err == syscall.EEXIST {
		// it's ok if the device already exists as long as config is similar
		existing, err := netlink.LinkByName(vxlan.Name)
		if err != nil {
			return nil, err
		}

		incompat := vxlanLinksIncompat(vxlan, existing)
		if incompat == "" {
			return existing.(*netlink.Vxlan), nil
		}

		// delete existing
		log.Warningf("%q already exists with incompatable configuration: %v; recreating device", vxlan.Name, incompat)
		if err = netlink.LinkDel(existing); err != nil {
			return nil, fmt.Errorf("failed to delete interface: %v", err)
		}

		// create new
		if err = netlink.LinkAdd(vxlan); err != nil {
			return nil, fmt.Errorf("failed to create vxlan interface: %v", err)
		}
	} else if err != nil {
		return nil, err
	}

	ifindex := vxlan.Index
	link, err := netlink.LinkByIndex(vxlan.Index)
	if err != nil {
		return nil, fmt.Errorf("can't locate created vxlan device with index %v", ifindex)
	}
	var ok bool
	if vxlan, ok = link.(*netlink.Vxlan); !ok {
		return nil, fmt.Errorf("created vxlan device with index %v is not vxlan", ifindex)
	}

	return vxlan, nil
}

func (u *NetLinkProbe) onLinkAdded(index int) {
	link, err := netlink.LinkByIndex(index)
	if err != nil {
		logging.GetLogger().Warningf("Failed to find interface %d: %s", index, err.Error())
		return
	}

	u.addLinkToTopology(link)
}

func GetBridgeFromIndex(idx int) (string, error) {
	var attr, bridge *netlink.LinkAttrs

	links, err := netlink.LinkList()
	if err != nil {
		glog.Error(err)
		return "", err
	}

	for _, link := range links {
		if link.Type() != "veth" {
			continue
		}

		if link.Attrs().Index == idx {
			attr = link.Attrs()
			break
		}
	}

	if attr == nil {
		return "", fmt.Errorf("cann't find nic whose ifindex is %d", idx)
	}

	for _, link := range links {
		if link.Type() != "bridge" && link.Type() != "openvswitch" {
			continue
		}

		if link.Attrs().Index == attr.MasterIndex {
			bridge = link.Attrs()
			break
		}
	}

	if bridge == nil {
		return "", fmt.Errorf("cann't find bridge contains nic whose ifindex is %d", idx)
	}

	if bridge.Name == "ovs-system" {
		veth, err := netlink.LinkByIndex(idx)
		if err != nil {
			return "", err
		}

		out, err := exec.Command("ovs-vsctl", "port-to-br", veth.Attrs().Name).CombinedOutput()
		if err != nil {
			return "", err
		}
		bridge.Name = strings.TrimSpace(string(out))
	}

	glog.Infof("find bridge %s", bridge.Name)

	return bridge.Name, nil
}

// Adds a macvlan interface to a container for use with the egress router feature
func addMacvlan(netns string) error {
	var defIface netlink.Link
	var err error

	// Find interface with the default route
	routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
	if err != nil {
		return fmt.Errorf("failed to read routes: %v", err)
	}

	for _, r := range routes {
		if r.Dst == nil {
			defIface, err = netlink.LinkByIndex(r.LinkIndex)
			if err != nil {
				return fmt.Errorf("failed to get default route interface: %v", err)
			}
		}
	}
	if defIface == nil {
		return fmt.Errorf("failed to find default route interface")
	}

	podNs, err := ns.GetNS(netns)
	if err != nil {
		return fmt.Errorf("could not open netns %q", netns)
	}
	defer podNs.Close()

	err = netlink.LinkAdd(&netlink.Macvlan{
		LinkAttrs: netlink.LinkAttrs{
			MTU:         defIface.Attrs().MTU,
			Name:        "macvlan0",
			ParentIndex: defIface.Attrs().Index,
			Namespace:   netlink.NsFd(podNs.Fd()),
		},
		Mode: netlink.MACVLAN_MODE_PRIVATE,
	})
	if err != nil {
		return fmt.Errorf("failed to create macvlan interface: %v", err)
	}
	return podNs.Do(func(netns ns.NetNS) error {
		l, err := netlink.LinkByName("macvlan0")
		if err != nil {
			return fmt.Errorf("failed to find macvlan interface: %v", err)
		}
		err = netlink.LinkSetUp(l)
		if err != nil {
			return fmt.Errorf("failed to set macvlan interface up: %v", err)
		}
		return nil
	})
}

func (u *NetLinkProbe) onLinkDeleted(index int) {
	logging.GetLogger().Debugf("Link %d deleted", index)

	u.Graph.Lock()
	defer u.Graph.Unlock()

	var intf *graph.Node

	intfs := u.Graph.LookupNodes(graph.Metadata{"IfIndex": int64(index)})
	switch l := len(intfs); {
	case l == 1:
		intf = intfs[0]
	case l > 1:
	Loop:
		for _, i := range intfs {
			parents := u.Graph.LookupParentNodes(i, nil)
			for _, parent := range parents {
				if parent.ID == u.Root.ID {
					intf = i
					break Loop
				}
			}
		}
	}

	// case of removing the interface from a bridge
	if intf != nil {
		parents := u.Graph.LookupParentNodes(intf, graph.Metadata{"Type": "bridge"})
		for _, parent := range parents {
			u.Graph.Unlink(parent, intf)
		}
	}

	// check wheter the interface has been deleted or not
	// we get a delete event when an interace is removed from a bridge
	_, err := netlink.LinkByIndex(index)
	if err != nil && intf != nil {
		if driver, ok := intf.Metadata()["Driver"]; ok {
			// if openvswitch do not remove let's do the job by ovs piece of code
			if driver == "openvswitch" {
				u.Graph.Unlink(u.Root, intf)
			} else {
				u.Graph.DelNode(intf)
			}
		}
	}

	delete(u.indexToChildrenQueue, int64(index))
}

func matchRoute(ifaceName string, dest net.IP) func(m syscall.NetlinkMessage) (bool, error) {
	return func(m syscall.NetlinkMessage) (bool, error) {
		switch m.Header.Type {
		case syscall.RTM_NEWROUTE:
			route, err := deserializeRoute(m.Data)
			if err != nil {
				return true, err
			}
			link, _ := netlink.LinkByIndex(route.LinkIndex)
			if link.Attrs().Name == ifaceName && route.Dst.IP.Equal(dest) {
				return true, nil
			}
		}
		return false, nil
	}
}

func resourceLXCBridgeRead(d *schema.ResourceData, meta interface{}) error {
	bridgeIndex, err := strconv.Atoi(d.Id())
	if err != nil {
		return fmt.Errorf("Internal error reading resource ID: %v", err)
	}

	bridge, err := netlink.LinkByIndex(bridgeIndex)
	if err != nil {
		return fmt.Errorf("Unable to find bridge %v: %v", bridgeIndex, err)
	}

	d.Set("mac", bridge.Attrs().HardwareAddr.String())

	log.Printf("[INFO] Bridge info: %v", bridge)

	return nil
}

func forEachRoute(ignoreIfaceNames map[string]struct{}, check func(netlink.Route) error) error {
	routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
	if err != nil {
		return err
	}
	for _, route := range routes {
		if link, err := netlink.LinkByIndex(route.LinkIndex); err == nil {
			if _, found := ignoreIfaceNames[link.Attrs().Name]; found {
				continue
			}
		}
		if err := check(route); err != nil {
			return err
		}
	}
	return nil
}

// Adds a macvlan interface to a container for use with the egress router feature
func addMacvlan(netns string) error {
	var defIface netlink.Link
	var err error

	// Find interface with the default route
	routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
	if err != nil {
		return fmt.Errorf("failed to read routes: %v", err)
	}

	for _, r := range routes {
		if r.Dst == nil {
			defIface, err = netlink.LinkByIndex(r.LinkIndex)
			if err != nil {
				return fmt.Errorf("failed to get default route interface: %v", err)
			}
		}
	}
	if defIface == nil {
		return fmt.Errorf("failed to find default route interface")
	}

	return ns.WithNetNSPath(netns, func(ns.NetNS) error {
		err := netlink.LinkAdd(&netlink.Macvlan{
			LinkAttrs: netlink.LinkAttrs{
				MTU:         defIface.Attrs().MTU,
				Name:        "macvlan0",
				ParentIndex: defIface.Attrs().Index,
			},
			Mode: netlink.MACVLAN_MODE_PRIVATE,
		})
		if err != nil {
			return fmt.Errorf("failed to create macvlan interface: %v", err)
		}
		l, err := netlink.LinkByName("macvlan0")
		if err != nil {
			return fmt.Errorf("failed to find macvlan interface: %v", err)
		}
		err = netlink.LinkSetUp(l)
		if err != nil {
			return fmt.Errorf("failed to set macvlan interface up: %v", err)
		}
		return nil
	})
}

func testAccCheckLXCBridgeDestroy(s *terraform.State) error {
	for _, rs := range s.RootModule().Resources {
		if rs.Type != "lxc_bridge" {
			continue
		}

		bridgeIndex, err := strconv.Atoi(rs.Primary.ID)
		if err != nil {
			return fmt.Errorf("Internal error reading resource ID.")
		}
		_, err = netlink.LinkByIndex(bridgeIndex)
		if err == nil {
			return fmt.Errorf("Bridge still exists.")
		}

	}

	return nil
}

// For a given container, returns host veth name, container veth MAC, and pod IP
func getVethInfo(netns, containerIfname string) (string, string, string, error) {
	var (
		peerIfindex int
		contVeth    netlink.Link
		err         error
		podIP       string
	)

	containerNs, err := ns.GetNS(netns)
	if err != nil {
		return "", "", "", fmt.Errorf("failed to get container netns: %v", err)
	}
	defer containerNs.Close()

	err = containerNs.Do(func(ns.NetNS) error {
		contVeth, err = netlink.LinkByName(containerIfname)
		if err != nil {
			return err
		}
		peerIfindex = contVeth.Attrs().ParentIndex

		addrs, err := netlink.AddrList(contVeth, syscall.AF_INET)
		if err != nil {
			return fmt.Errorf("failed to get container IP addresses: %v", err)
		}
		if len(addrs) == 0 {
			return fmt.Errorf("container had no addresses")
		}
		podIP = addrs[0].IP.String()

		return nil
	})
	if err != nil {
		return "", "", "", fmt.Errorf("failed to inspect container interface: %v", err)
	}

	hostVeth, err := netlink.LinkByIndex(peerIfindex)
	if err != nil {
		return "", "", "", fmt.Errorf("failed to get host veth: %v", err)
	}

	return hostVeth.Attrs().Name, contVeth.Attrs().HardwareAddr.String(), podIP, nil
}

func (mapper *NetLinkMapper) cacheUpdater() {
	logging.GetLogger().Debug("Start NetLink cache updater")

	var ifIndex uint32
	for {
		ifIndex = <-mapper.cacheUpdaterChan

		logging.GetLogger().Debug("ifIndex request received: %s", ifIndex)

		var attrs *netlink.LinkAttrs

		link, err := netlink.LinkByIndex(int(ifIndex))
		if err != nil {
			logging.GetLogger().Error("Error while getting interface by index via netling: ", err.Error())
			attrs = &netlink.LinkAttrs{}
		} else {
			attrs = link.Attrs()
		}

		mapper.cache.Set(strconv.Itoa(int(ifIndex)), attrs, cache.DefaultExpiration)
	}
}

// Wait for an interface to come up and have a route added to the multicast subnet.
// This matches the behaviour in 'weave attach', which is the only context in which
// we expect this to be called.  If you change one, change the other to match.
func EnsureInterfaceAndMcastRoute(ifaceName string) (*net.Interface, error) {
	iface, err := ensureInterface(ifaceName)
	if err != nil {
		return nil, err
	}
	dest := net.IPv4(224, 0, 0, 0)
	if CheckRouteExists(ifaceName, dest) {
		return iface, err
	}
	ch := make(chan netlink.RouteUpdate)
	done := make(chan struct{})
	defer close(done)
	if err := netlink.RouteSubscribe(ch, done); err != nil {
		return nil, err
	}
	for update := range ch {
		link, _ := netlink.LinkByIndex(update.Route.LinkIndex)
		if link.Attrs().Name == ifaceName && update.Route.Dst.IP.Equal(dest) {
			break
		}
	}
	return iface, nil
}

func (nm *NetlinkMonitor) handleMasterChange(node *ExtInterface, link netlink.Link, isAdd bool) error {
	newMasterIdx := link.Attrs().MasterIndex
	Debug.Printf("link %s master %d\n", node.Link().Attrs().Name, newMasterIdx)
	if newMasterIdx != node.Link().Attrs().MasterIndex || isAdd {
		if newMasterIdx != 0 {
			// add case
			masterLink, err := netlink.LinkByIndex(newMasterIdx)
			if err != nil {
				return err
			}
			bridge, ok := masterLink.(*netlink.Bridge)
			if !ok {
				return fmt.Errorf("unsupported non-bridge master")
			}
			master := nm.ensureBridge(bridge)
			if node.ID() < 0 {
				node.SetID(nm.g.NewNodeID())
				nm.g.AddNode(node)
			}
			// set to 0 so that the normal egress path is taken
			fid, tid := 0, 0
			nm.g.SetEdge(canvas.NewEdgeChain(node, master, &fid, &tid))
			nm.g.SetEdge(canvas.NewEdgeChain(master, node, &tid, &fid))
			if err := nm.r.Provision(nm.g, []InterfaceNode{node}); err != nil {
				return err
			}
			if err := nm.ensureInterface(nm.g, node); err != nil {
				return err
			}
			nm.r.Run(nm.g, []InterfaceNode{node})
		} else {
			// remove case
		}
	}
	return nil
}

func (*nl) LinkByIndex(index int) (netlink.Link, error) {
	return netlink.LinkByIndex(index)
}

// Set up all networking (host/container veth, OVS flows, IPAM, loopback, etc)
func (m *podManager) setup(req *cniserver.PodRequest) (*cnitypes.Result, *kubehostport.RunningPod, error) {
	podConfig, pod, err := m.getPodConfig(req)
	if err != nil {
		return nil, nil, err
	}

	ipamResult, err := m.ipamAdd(req.Netns, req.ContainerId)
	if err != nil {
		// TODO: Remove this hack once we've figured out how to retrieve the netns
		// of an exited container. Currently, restarting docker will leak a bunch of
		// ips. This will exhaust available ip space unless we cleanup old ips. At the
		// same time we don't want to try GC'ing them periodically as that could lead
		// to a performance regression in starting pods. So on each setup failure, try
		// GC on the assumption that the kubelet is going to retry pod creation, and
		// when it does, there will be ips.
		m.ipamGarbageCollection()

		return nil, nil, fmt.Errorf("failed to run IPAM for %v: %v", req.ContainerId, err)
	}
	podIP := ipamResult.IP4.IP.IP

	// Release any IPAM allocations and hostports if the setup failed
	var success bool
	defer func() {
		if !success {
			m.ipamDel(req.ContainerId)
			if err := m.hostportHandler.SyncHostports(TUN, m.getRunningPods()); err != nil {
				glog.Warningf("failed syncing hostports: %v", err)
			}
		}
	}()

	// Open any hostports the pod wants
	newPod := &kubehostport.RunningPod{Pod: pod, IP: podIP}
	if err := m.hostportHandler.OpenPodHostportsAndSync(newPod, TUN, m.getRunningPods()); err != nil {
		return nil, nil, err
	}

	var hostVeth, contVeth netlink.Link
	err = ns.WithNetNSPath(req.Netns, func(hostNS ns.NetNS) error {
		hostVeth, contVeth, err = ip.SetupVeth(podInterfaceName, int(m.mtu), hostNS)
		if err != nil {
			return fmt.Errorf("failed to create container veth: %v", err)
		}
		// refetch to get hardware address and other properties
		contVeth, err = netlink.LinkByIndex(contVeth.Attrs().Index)
		if err != nil {
			return fmt.Errorf("failed to fetch container veth: %v", err)
		}

		// Clear out gateway to prevent ConfigureIface from adding the cluster
		// subnet via the gateway
		ipamResult.IP4.Gateway = nil
		if err = ipam.ConfigureIface(podInterfaceName, ipamResult); err != nil {
			return fmt.Errorf("failed to configure container IPAM: %v", err)
		}

		lo, err := netlink.LinkByName("lo")
		if err == nil {
			err = netlink.LinkSetUp(lo)
		}
		if err != nil {
			return fmt.Errorf("failed to configure container loopback: %v", err)
		}
		return nil
	})
	if err != nil {
		return nil, nil, err
	}

	if podConfig.wantMacvlan {
		if err := addMacvlan(req.Netns); err != nil {
			return nil, nil, err
		}
	}

	contVethMac := contVeth.Attrs().HardwareAddr.String()
	vnidStr := vnidToString(podConfig.vnid)
	out, err := exec.Command(sdnScript, setUpCmd, hostVeth.Attrs().Name, contVethMac, podIP.String(), vnidStr, podConfig.ingressBandwidth, podConfig.egressBandwidth).CombinedOutput()
	glog.V(5).Infof("SetUpPod network plugin output: %s, %v", string(out), err)

	if isScriptError(err) {
		return nil, nil, fmt.Errorf("error running network setup script:\nhostVethName %s, contVethMac %s, podIP %s, podConfig %#v\n %s", hostVeth.Attrs().Name, contVethMac, podIP.String(), podConfig, getScriptError(out))
	} else if err != nil {
		return nil, nil, err
	}

	success = true
	return ipamResult, newPod, nil
}

// Set up all networking (host/container veth, OVS flows, IPAM, loopback, etc)
func (m *podManager) setup(req *cniserver.PodRequest) (*cnitypes.Result, *kubehostport.RunningPod, error) {
	podConfig, pod, err := m.getPodConfig(req)
	if err != nil {
		return nil, nil, err
	}

	ipamResult, err := m.runIPAM(req.Netns, cniserver.CNI_ADD, req.ContainerId)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to run IPAM for %v: %v", req.ContainerId, err)
	}
	podIP := ipamResult.IP4.IP.IP

	// Release any IPAM allocations and hostports if the setup failed
	var success bool
	defer func() {
		if !success {
			m.runIPAM(req.Netns, cniserver.CNI_DEL, req.ContainerId)
			if err := m.hostportHandler.SyncHostports(TUN, m.getRunningPods()); err != nil {
				glog.Warningf("failed syncing hostports: %v", err)
			}
		}
	}()

	// Open any hostports the pod wants
	newPod := &kubehostport.RunningPod{Pod: pod, IP: podIP}
	if err := m.hostportHandler.OpenPodHostportsAndSync(newPod, TUN, m.getRunningPods()); err != nil {
		return nil, nil, err
	}

	var hostVeth, contVeth netlink.Link
	err = ns.WithNetNSPath(req.Netns, func(hostNS ns.NetNS) error {
		hostVeth, contVeth, err = ip.SetupVeth(podInterfaceName, int(m.mtu), hostNS)
		if err != nil {
			return fmt.Errorf("failed to create container veth: %v", err)
		}
		// refetch to get hardware address and other properties
		contVeth, err = netlink.LinkByIndex(contVeth.Attrs().Index)
		if err != nil {
			return fmt.Errorf("failed to fetch container veth: %v", err)
		}

		// Clear out gateway to prevent ConfigureIface from adding the cluster
		// subnet via the gateway
		ipamResult.IP4.Gateway = nil
		if err = ipam.ConfigureIface(podInterfaceName, ipamResult); err != nil {
			return fmt.Errorf("failed to configure container IPAM: %v", err)
		}

		lo, err := netlink.LinkByName("lo")
		if err == nil {
			err = netlink.LinkSetUp(lo)
		}
		if err != nil {
			return fmt.Errorf("failed to configure container loopback: %v", err)
		}
		return nil
	})
	if err != nil {
		return nil, nil, err
	}

	if podConfig.wantMacvlan {
		if err := addMacvlan(req.Netns); err != nil {
			return nil, nil, err
		}
	}

	contVethMac := contVeth.Attrs().HardwareAddr.String()
	vnidStr := vnidToString(podConfig.vnid)
	out, err := exec.Command(sdnScript, setUpCmd, hostVeth.Attrs().Name, contVethMac, podIP.String(), vnidStr, podConfig.ingressBandwidth, podConfig.egressBandwidth).CombinedOutput()
	glog.V(5).Infof("SetUpPod network plugin output: %s, %v", string(out), err)

	if isScriptError(err) {
		return nil, nil, fmt.Errorf("error running network setup script:\nhostVethName %s, contVethMac %s, podIP %s, podConfig %#v\n %s", hostVeth.Attrs().Name, contVethMac, podIP.String(), podConfig, getScriptError(out))
	} else if err != nil {
		return nil, nil, err
	}

	success = true
	return ipamResult, newPod, nil
}