// GC enters the pod by fork/exec()ing the stage1's /gc similar to /init.
// /gc can expect to have its CWD set to the pod root.
func GC(pdir string, uuid *types.UUID) error {
	err := unregisterPod(pdir, uuid)
	if err != nil {
		// Probably not worth abandoning the rest
		log.PrintE("warning: could not unregister pod with metadata service", err)
	}

	stage1Path := common.Stage1RootfsPath(pdir)

	ep, err := getStage1Entrypoint(pdir, gcEntrypoint)
	if err != nil {
		return errwrap.Wrap(errors.New("error determining 'gc' entrypoint"), err)
	}

	args := []string{filepath.Join(stage1Path, ep)}
	if debugEnabled {
		args = append(args, "--debug")
	}
	args = append(args, uuid.String())

	c := exec.Cmd{
		Path:   args[0],
		Args:   args,
		Stderr: os.Stderr,
		Dir:    pdir,
	}
	return c.Run()
}

func StopPod(dir string, force bool, uuid *types.UUID) error {
	s1rootfs := common.Stage1RootfsPath(dir)

	if err := os.Chdir(dir); err != nil {
		return fmt.Errorf("failed changing to dir: %v", err)
	}

	ep, err := getStage1Entrypoint(dir, stopEntrypoint)
	if err != nil {
		return fmt.Errorf("rkt stop not implemented for pod's stage1: %v", err)
	}
	args := []string{filepath.Join(s1rootfs, ep)}
	debug("Execing %s", ep)

	if force {
		args = append(args, "--force")
	}

	args = append(args, uuid.String())

	c := exec.Cmd{
		Path:   args[0],
		Args:   args,
		Stdout: os.Stdout,
		Stderr: os.Stderr,
	}

	return c.Run()
}

// registerPod registers pod with metadata service.
// Returns authentication token to be passed in the URL
func registerPod(root string, uuid *types.UUID, apps schema.AppList) (token string, rerr error) {
	u := uuid.String()

	var err error
	token, err = generateMDSToken()
	if err != nil {
		rerr = errwrap.Wrap(errors.New("failed to generate MDS token"), err)
		return
	}

	pmfPath := common.PodManifestPath(root)
	pmf, err := os.Open(pmfPath)
	if err != nil {
		rerr = errwrap.Wrap(fmt.Errorf("failed to open runtime manifest (%v)", pmfPath), err)
		return
	}

	pth := fmt.Sprintf("/pods/%v?token=%v", u, token)
	err = httpRequest("PUT", pth, pmf)
	pmf.Close()
	if err != nil {
		rerr = errwrap.Wrap(errors.New("failed to register pod with metadata svc"), err)
		return
	}

	defer func() {
		if rerr != nil {
			unregisterPod(root, uuid)
		}
	}()

	rf, err := os.Create(filepath.Join(root, mdsRegisteredFile))
	if err != nil {
		rerr = errwrap.Wrap(errors.New("failed to create mds-register file"), err)
		return
	}
	rf.Close()

	for _, app := range apps {
		ampath := common.ImageManifestPath(root, app.Name)
		amf, err := os.Open(ampath)
		if err != nil {
			rerr = errwrap.Wrap(fmt.Errorf("failed reading app manifest %q", ampath), err)
			return
		}

		err = registerApp(u, app.Name.String(), amf)
		amf.Close()
		if err != nil {
			rerr = errwrap.Wrap(errors.New("failed to register app with metadata svc"), err)
			return
		}
	}

	return
}

// setupTapDevice creates persistent macvtap device
// and returns a newly created netlink.Link structure
// using part of pod hash and interface number in interface name
func setupMacVTapDevice(podID types.UUID, config MacVTapNetConf, interfaceNumber int) (netlink.Link, error) {
	master, err := netlink.LinkByName(config.Master)
	if err != nil {
		return nil, errwrap.Wrap(fmt.Errorf("cannot find master device '%v'", config.Master), err)
	}
	var mode netlink.MacvlanMode
	switch config.Mode {
	// if not set - defaults to bridge mode as in:
	// https://github.com/coreos/rkt/blob/master/Documentation/networking.md#macvlan
	case "", "bridge":
		mode = netlink.MACVLAN_MODE_BRIDGE
	case "private":
		mode = netlink.MACVLAN_MODE_PRIVATE
	case "vepa":
		mode = netlink.MACVLAN_MODE_VEPA
	case "passthru":
		mode = netlink.MACVLAN_MODE_PASSTHRU
	default:
		return nil, fmt.Errorf("unsupported macvtap mode: %v", config.Mode)
	}
	mtu := master.Attrs().MTU
	if config.MTU != 0 {
		mtu = config.MTU
	}
	interfaceName := fmt.Sprintf("rkt-%s-vtap%d", podID.String()[0:4], interfaceNumber)
	link := &netlink.Macvtap{
		Macvlan: netlink.Macvlan{
			LinkAttrs: netlink.LinkAttrs{
				Name:        interfaceName,
				MTU:         mtu,
				ParentIndex: master.Attrs().Index,
			},
			Mode: mode,
		},
	}

	if err := netlink.LinkAdd(link); err != nil {
		return nil, errwrap.Wrap(errors.New("cannot create macvtap interface"), err)
	}

	// TODO: duplicate following lines for ipv6 support, when it will be added in other places
	ipv4SysctlValueName := fmt.Sprintf(IPv4InterfaceArpProxySysctlTemplate, interfaceName)
	if _, err := cnisysctl.Sysctl(ipv4SysctlValueName, "1"); err != nil {
		// remove the newly added link and ignore errors, because we already are in a failed state
		_ = netlink.LinkDel(link)
		return nil, errwrap.Wrap(fmt.Errorf("failed to set proxy_arp on newly added interface %q", interfaceName), err)
	}

	if err := netlink.LinkSetUp(link); err != nil {
		// remove the newly added link and ignore errors, because we already are in a failed state
		_ = netlink.LinkDel(link)
		return nil, errwrap.Wrap(errors.New("cannot set up macvtap interface"), err)
	}
	return link, nil
}

func removeJournalLink(uuid *types.UUID) error {
	// if the host runs systemd, we link the journal and set pod's machine-id
	// as pod's UUID without the dashes in init.go:
	// https://github.com/coreos/rkt/blob/95e6bc/stage1/init/init.go#L382
	machineID := strings.Replace(uuid.String(), "-", "", -1)
	journalLink := filepath.Join("/var/log/journal", machineID)
	err := os.Remove(journalLink)
	if err != nil && err.(*os.PathError).Err == syscall.ENOENT {
		err = nil
	}
	return err
}

// unregisterPod unregisters pod with the metadata service.
func unregisterPod(root string, uuid *types.UUID) error {
	_, err := os.Stat(filepath.Join(root, mdsRegisteredFile))
	switch {
	case err == nil:
		pth := path.Join("/pods", uuid.String())
		return httpRequest("DELETE", pth, nil)

	case os.IsNotExist(err):
		return nil

	default:
		return err
	}
}

// setupTapDevice creates persistent tap device
// and returns a newly created netlink.Link structure
func setupTapDevice(podID types.UUID) (netlink.Link, error) {
	// network device names are limited to 16 characters
	// the suffix %d will be replaced by the kernel with a suitable number
	nameTemplate := fmt.Sprintf("rkt-%s-tap%%d", podID.String()[0:4])
	ifName, err := tuntap.CreatePersistentIface(nameTemplate, tuntap.Tap)
	if err != nil {
		return nil, errwrap.Wrap(errors.New("tuntap persist"), err)
	}

	link, err := netlink.LinkByName(ifName)
	if err != nil {
		return nil, errwrap.Wrap(fmt.Errorf("cannot find link %q", ifName), err)
	}

	if err := netlink.LinkSetUp(link); err != nil {
		return nil, errwrap.Wrap(fmt.Errorf("cannot set link up %q", ifName), err)
	}
	return link, nil
}

// setupTapDevice creates persistent macvtap device
// and returns a newly created netlink.Link structure
func setupMacVTapDevice(podID types.UUID, config MacVTapNetConf) (netlink.Link, error) {
	master, err := netlink.LinkByName(config.Master)
	if err != nil {
		return nil, errwrap.Wrap(fmt.Errorf("cannot find master device '%v'", config.Master), err)
	}
	var mode netlink.MacvlanMode
	switch config.Mode {
	// if not set - defaults to bridge mode as in:
	// https://github.com/coreos/rkt/blob/master/Documentation/networking.md#macvlan
	case "", "bridge":
		mode = netlink.MACVLAN_MODE_BRIDGE
	case "private":
		mode = netlink.MACVLAN_MODE_PRIVATE
	case "vepa":
		mode = netlink.MACVLAN_MODE_VEPA
	case "passthru":
		mode = netlink.MACVLAN_MODE_PASSTHRU
	default:
		return nil, fmt.Errorf("unsupported macvtap mode: %v", config.Mode)
	}
	mtu := master.Attrs().MTU
	if config.MTU != 0 {
		mtu = config.MTU
	}
	nameTemplate := fmt.Sprintf("rkt-%s-vtap%%d", podID.String()[0:4])
	link := &netlink.Macvtap{
		Macvlan: netlink.Macvlan{
			LinkAttrs: netlink.LinkAttrs{
				Name:        nameTemplate,
				MTU:         mtu,
				ParentIndex: master.Attrs().Index,
			},
			Mode: mode,
		},
	}

	if err := netlink.LinkAdd(link); err != nil {
		return nil, errwrap.Wrap(errors.New("cannot create macvtap interface"), err)
	}
	return link, nil
}

// GC enters the pod by fork/exec()ing the stage1's /gc similar to /init.
// /gc can expect to have its CWD set to the pod root.
func GC(pdir string, uuid *types.UUID, localConfig string) error {
	err := unregisterPod(pdir, uuid)
	if err != nil {
		// Probably not worth abandoning the rest
		log.PrintE("warning: could not unregister pod with metadata service", err)
	}

	stage1Path := common.Stage1RootfsPath(pdir)
	s1v, err := getStage1InterfaceVersion(pdir)
	if err != nil {
		return errwrap.Wrap(errors.New("Could not determine stage1 version"), err)
	}

	ep, err := getStage1Entrypoint(pdir, gcEntrypoint)
	if err != nil {
		return errwrap.Wrap(errors.New("error determining 'gc' entrypoint"), err)
	}

	args := []string{filepath.Join(stage1Path, ep)}
	if debugEnabled {
		args = append(args, "--debug")
	}
	if interfaceVersionSupportsGCLocalConfig(s1v) && localConfig != "" {
		args = append(args, "--local-config", localConfig)
	}
	args = append(args, uuid.String())

	debug("Execing %v", args)
	c := exec.Cmd{
		Path:   args[0],
		Args:   args,
		Stderr: os.Stderr,
		Dir:    pdir,
	}
	return c.Run()
}

// WriteUUIDToFile writes the uuid string to the given path.
func WriteUUIDToFile(uuid *types.UUID, path string) error {
	return ioutil.WriteFile(path, []byte(uuid.String()), 0644)
}

// TODO(iaguis): RmConfig?
func RmApp(dir string, uuid *types.UUID, usesOverlay bool, appName *types.ACName, podPID int) error {
	p, err := stage1types.LoadPod(dir, uuid)
	if err != nil {
		return errwrap.Wrap(errors.New("error loading pod manifest"), err)
	}

	pm := p.Manifest

	var mutable bool
	ms, ok := pm.Annotations.Get("coreos.com/rkt/stage1/mutable")
	if ok {
		mutable, err = strconv.ParseBool(ms)
		if err != nil {
			return errwrap.Wrap(errors.New("error parsing mutable annotation"), err)
		}
	}

	if !mutable {
		return errors.New("immutable pod: cannot remove application")
	}

	app := pm.Apps.Get(*appName)
	if app == nil {
		return fmt.Errorf("error: nonexistent app %q", *appName)
	}

	treeStoreID, err := ioutil.ReadFile(common.AppTreeStoreIDPath(dir, *appName))
	if err != nil {
		return err
	}

	eep, err := getStage1Entrypoint(dir, enterEntrypoint)
	if err != nil {
		return errwrap.Wrap(errors.New("error determining 'enter' entrypoint"), err)
	}

	args := []string{
		uuid.String(),
		appName.String(),
		filepath.Join(common.Stage1RootfsPath(dir), eep),
		strconv.Itoa(podPID),
	}

	if err := callEntrypoint(dir, appStopEntrypoint, args); err != nil {
		return err
	}

	if err := callEntrypoint(dir, appRmEntrypoint, args); err != nil {
		return err
	}

	appInfoDir := common.AppInfoPath(dir, *appName)
	if err := os.RemoveAll(appInfoDir); err != nil {
		return errwrap.Wrap(errors.New("error removing app info directory"), err)
	}

	if usesOverlay {
		appRootfs := common.AppRootfsPath(dir, *appName)
		if err := syscall.Unmount(appRootfs, 0); err != nil {
			return err
		}

		ts := filepath.Join(dir, "overlay", string(treeStoreID))
		if err := os.RemoveAll(ts); err != nil {
			return errwrap.Wrap(errors.New("error removing app info directory"), err)
		}
	}

	if err := os.RemoveAll(common.AppPath(dir, *appName)); err != nil {
		return err
	}

	appStatusPath := filepath.Join(common.Stage1RootfsPath(dir), "rkt", "status", appName.String())
	if err := os.Remove(appStatusPath); err != nil && !os.IsNotExist(err) {
		return err
	}

	envPath := filepath.Join(common.Stage1RootfsPath(dir), "rkt", "env", appName.String())
	if err := os.Remove(envPath); err != nil && !os.IsNotExist(err) {
		return err
	}

	removeAppFromPodManifest(pm, appName)

	if err := updatePodManifest(dir, pm); err != nil {
		return err
	}

	return nil
}

// kvmSetup prepare new Networking to be used in kvm environment based on tuntap pair interfaces
// to allow communication with virtual machine created by lkvm tool
func kvmSetup(podRoot string, podID types.UUID, fps []ForwardedPort, netList common.NetList, localConfig string, noDNS bool) (*Networking, error) {
	network := Networking{
		podEnv: podEnv{
			podRoot:      podRoot,
			podID:        podID,
			netsLoadList: netList,
			localConfig:  localConfig,
		},
	}
	var e error

	// If there's a network set as default in CNI configuration
	defaultGatewaySet := false

	_, defaultNet, err := net.ParseCIDR("0.0.0.0/0")
	if err != nil {
		return nil, errwrap.Wrap(errors.New("error when parsing net address"), err)
	}

	network.nets, e = network.loadNets()
	if e != nil {
		return nil, errwrap.Wrap(errors.New("error loading network definitions"), e)
	}

	// did stage0 already make /etc/rkt-resolv.conf (i.e. --dns passed)
	resolvPath := filepath.Join(common.Stage1RootfsPath(podRoot), "etc/rkt-resolv.conf")
	_, err = os.Stat(resolvPath)
	if err != nil && !os.IsNotExist(err) {
		return nil, errwrap.Wrap(fmt.Errorf("error statting /etc/rkt-resolv.conf"), err)
	}
	podHasResolvConf := err == nil

	for i, n := range network.nets {
		if n.conf.Type == "flannel" {
			if err := kvmTransformFlannelNetwork(&n); err != nil {
				return nil, errwrap.Wrap(errors.New("cannot transform flannel network into basic network"), err)
			}
		}
		switch n.conf.Type {
		case "ptp":
			link, err := setupTapDevice(podID)
			if err != nil {
				return nil, err
			}
			ifName := link.Attrs().Name
			n.runtime.IfName = ifName

			err = kvmSetupNetAddressing(&network, n, ifName)
			if err != nil {
				return nil, err
			}

			// add address to host tap device
			err = ensureHasAddr(
				link,
				&net.IPNet{
					IP:   n.runtime.IP4.Gateway,
					Mask: net.IPMask(n.runtime.Mask),
				},
			)
			if err != nil {
				return nil, errwrap.Wrap(fmt.Errorf("cannot add address to host tap device %q", ifName), err)
			}

			if err := removeAllRoutesOnLink(link); err != nil {
				return nil, errwrap.Wrap(fmt.Errorf("cannot remove route on host tap device %q", ifName), err)
			}

			if err := addRoute(link, n.runtime.IP); err != nil {
				return nil, errwrap.Wrap(errors.New("cannot add on host direct route to pod"), err)
			}

		case "bridge":
			config := BridgeNetConf{
				NetConf: NetConf{
					MTU: defaultMTU,
				},
				BrName: defaultBrName,
			}
			if err := json.Unmarshal(n.confBytes, &config); err != nil {
				return nil, errwrap.Wrap(fmt.Errorf("error parsing %q result", n.conf.Name), err)
			}

			br, err := ensureBridgeIsUp(config.BrName, config.MTU)
			if err != nil {
				return nil, errwrap.Wrap(errors.New("error in time of bridge setup"), err)
			}
			link, err := setupTapDevice(podID)
			if err != nil {
				return nil, errwrap.Wrap(errors.New("can not setup tap device"), err)
			}
			err = netlink.LinkSetMaster(link, br)
			if err != nil {
				rErr := tuntap.RemovePersistentIface(n.runtime.IfName, tuntap.Tap)
				if rErr != nil {
					stderr.PrintE("warning: could not cleanup tap interface", rErr)
				}
				return nil, errwrap.Wrap(errors.New("can not add tap interface to bridge"), err)
//.........这里部分代码省略.........

// kvmSetup prepare new Networking to be used in kvm environment based on tuntap pair interfaces
// to allow communication with virtual machine created by lkvm tool
func kvmSetup(podRoot string, podID types.UUID, fps []ForwardedPort, netList common.NetList, localConfig string) (*Networking, error) {
	network := Networking{
		podEnv: podEnv{
			podRoot:      podRoot,
			podID:        podID,
			netsLoadList: netList,
			localConfig:  localConfig,
		},
	}
	var e error

	// If there's a network set as default in CNI configuration
	defaultGatewaySet := false

	_, defaultNet, err := net.ParseCIDR("0.0.0.0/0")
	if err != nil {
		return nil, errwrap.Wrap(errors.New("error when parsing net address"), err)
	}

	network.nets, e = network.loadNets()
	if e != nil {
		return nil, errwrap.Wrap(errors.New("error loading network definitions"), e)
	}

	for i, n := range network.nets {
		if n.conf.Type == "flannel" {
			if err := kvmTransformFlannelNetwork(&n); err != nil {
				return nil, errwrap.Wrap(errors.New("cannot transform flannel network into basic network"), err)
			}
		}
		switch n.conf.Type {
		case "ptp":
			link, err := setupTapDevice(podID)
			if err != nil {
				return nil, err
			}
			ifName := link.Attrs().Name
			n.runtime.IfName = ifName

			err = kvmSetupNetAddressing(&network, n, ifName)
			if err != nil {
				return nil, err
			}

			// add address to host tap device
			err = ensureHasAddr(
				link,
				&net.IPNet{
					IP:   n.runtime.IP4.Gateway,
					Mask: net.IPMask(n.runtime.Mask),
				},
			)
			if err != nil {
				return nil, errwrap.Wrap(fmt.Errorf("cannot add address to host tap device %q", ifName), err)
			}

			if err := removeAllRoutesOnLink(link); err != nil {
				return nil, errwrap.Wrap(fmt.Errorf("cannot remove route on host tap device %q", ifName), err)
			}

			if err := addRoute(link, n.runtime.IP); err != nil {
				return nil, errwrap.Wrap(errors.New("cannot add on host direct route to pod"), err)
			}

		case "bridge":
			config := BridgeNetConf{
				NetConf: NetConf{
					MTU: defaultMTU,
				},
				BrName: defaultBrName,
			}
			if err := json.Unmarshal(n.confBytes, &config); err != nil {
				return nil, errwrap.Wrap(fmt.Errorf("error parsing %q result", n.conf.Name), err)
			}

			br, err := ensureBridgeIsUp(config.BrName, config.MTU)
			if err != nil {
				return nil, errwrap.Wrap(errors.New("error in time of bridge setup"), err)
			}
			link, err := setupTapDevice(podID)
			if err != nil {
				return nil, errwrap.Wrap(errors.New("can not setup tap device"), err)
			}
			err = netlink.LinkSetMaster(link, br)
			if err != nil {
				rErr := tuntap.RemovePersistentIface(n.runtime.IfName, tuntap.Tap)
				if rErr != nil {
					stderr.PrintE("warning: could not cleanup tap interface", rErr)
				}
				return nil, errwrap.Wrap(errors.New("can not add tap interface to bridge"), err)
			}

			ifName := link.Attrs().Name
			n.runtime.IfName = ifName

			err = kvmSetupNetAddressing(&network, n, ifName)
			if err != nil {
				return nil, err
//.........这里部分代码省略.........

// kvmSetup prepare new Networking to be used in kvm environment based on tuntap pair interfaces
// to allow communication with virtual machine created by lkvm tool
func kvmSetup(podRoot string, podID types.UUID, fps []ForwardedPort, netList common.NetList, localConfig string) (*Networking, error) {
	network := Networking{
		podEnv: podEnv{
			podRoot:      podRoot,
			podID:        podID,
			netsLoadList: netList,
			localConfig:  localConfig,
		},
	}
	var e error
	network.nets, e = network.loadNets()
	if e != nil {
		return nil, errwrap.Wrap(errors.New("error loading network definitions"), e)
	}

	for i, n := range network.nets {
		if n.conf.Type == "flannel" {
			if err := kvmTransformFlannelNetwork(&n); err != nil {
				return nil, errwrap.Wrap(errors.New("cannot transform flannel network into basic network"), err)
			}
		}
		switch n.conf.Type {
		case "ptp":
			link, err := setupTapDevice(podID)
			if err != nil {
				return nil, err
			}
			ifName := link.Attrs().Name
			n.runtime.IfName = ifName

			err = kvmSetupNetAddressing(&network, n, ifName)
			if err != nil {
				return nil, err
			}

			// add address to host tap device
			err = ensureHasAddr(
				link,
				&net.IPNet{
					IP:   n.runtime.IP4.Gateway,
					Mask: net.IPMask(n.runtime.Mask),
				},
			)
			if err != nil {
				return nil, errwrap.Wrap(fmt.Errorf("cannot add address to host tap device %q", ifName), err)
			}

			if err := removeAllRoutesOnLink(link); err != nil {
				return nil, errwrap.Wrap(fmt.Errorf("cannot remove route on host tap device %q", ifName), err)
			}

			if err := addRoute(link, n.runtime.IP); err != nil {
				return nil, errwrap.Wrap(errors.New("cannot add on host direct route to pod"), err)
			}

		case "bridge":
			config := BridgeNetConf{
				NetConf: NetConf{
					MTU: defaultMTU,
				},
				BrName: defaultBrName,
			}
			if err := json.Unmarshal(n.confBytes, &config); err != nil {
				return nil, errwrap.Wrap(fmt.Errorf("error parsing %q result", n.conf.Name), err)
			}

			br, err := ensureBridgeIsUp(config.BrName, config.MTU)
			if err != nil {
				return nil, errwrap.Wrap(errors.New("error in time of bridge setup"), err)
			}
			link, err := setupTapDevice(podID)
			if err != nil {
				return nil, errwrap.Wrap(errors.New("can not setup tap device"), err)
			}
			err = netlink.LinkSetMaster(link, br)
			if err != nil {
				rErr := tuntap.RemovePersistentIface(n.runtime.IfName, tuntap.Tap)
				if rErr != nil {
					stderr.PrintE("warning: could not cleanup tap interface", rErr)
				}
				return nil, errwrap.Wrap(errors.New("can not add tap interface to bridge"), err)
			}

			ifName := link.Attrs().Name
			n.runtime.IfName = ifName

			err = kvmSetupNetAddressing(&network, n, ifName)
			if err != nil {
				return nil, err
			}

			if config.IsGw {
				err = ensureHasAddr(
					br,
					&net.IPNet{
						IP:   n.runtime.IP4.Gateway,
						Mask: net.IPMask(n.runtime.Mask),
					},
//.........这里部分代码省略.........