// findBinPath takes a binary path and returns a the absolute path of the
// binary relative to the app rootfs. This can be passed to ExecStart on the
// app's systemd service file directly.
func findBinPath(p *stage1commontypes.Pod, appName types.ACName, app types.App, workDir string, bin string) (string, error) {
	var binPath string
	switch {
	// absolute path, just use it
	case filepath.IsAbs(bin):
		binPath = bin
	// non-absolute path containing a slash, look in the working dir
	case strings.Contains(bin, "/"):
		binPath = filepath.Join(workDir, bin)
	// filename, search in the app's $PATH
	default:
		absRoot, err := filepath.Abs(p.Root)
		if err != nil {
			return "", errwrap.Wrap(errors.New("could not get pod's root absolute path"), err)
		}
		appRootfs := common.AppRootfsPath(absRoot, appName)
		appPathDirs := appSearchPaths(p, workDir, app)
		appPath := strings.Join(appPathDirs, ":")

		binPath, err = lookupPathInsideApp(bin, appPath, appRootfs, workDir)
		if err != nil {
			return "", errwrap.Wrap(fmt.Errorf("error looking up %q", bin), err)
		}
	}

	return binPath, nil
}

func appHasMountpoints(podPath string, appName types.ACName) (bool, error) {
	appRootfs := common.AppRootfsPath(podPath, appName)
	// add a filepath separator so we don't match the appRootfs path
	appRootfs += string(filepath.Separator)

	mi, err := os.Open("/proc/self/mountinfo")
	if err != nil {
		return false, err
	}
	defer mi.Close()

	sc := bufio.NewScanner(mi)
	for sc.Scan() {
		line := sc.Text()
		lineResult := strings.Split(line, " ")
		if len(lineResult) < 7 {
			return false, fmt.Errorf("not enough fields from line %q: %+v", line, lineResult)
		}

		mp := lineResult[4]

		if strings.HasPrefix(mp, appRootfs) {
			return true, nil
		}
	}
	if err := sc.Err(); err != nil {
		return false, err
	}
	return false, nil
}

// FindBinPath takes a binary path and returns a the absolute path of the
// binary relative to the app rootfs. This can be passed to ExecStart on the
// app's systemd service file directly.
func FindBinPath(p *stage1commontypes.Pod, ra *schema.RuntimeApp) (string, error) {
	if len(ra.App.Exec) == 0 {
		return "", errors.New("app has no executable")
	}

	bin := ra.App.Exec[0]

	var binPath string
	switch {
	// absolute path, just use it
	case filepath.IsAbs(bin):
		binPath = bin
	// non-absolute path containing a slash, look in the working dir
	case strings.Contains(bin, "/"):
		binPath = filepath.Join(ra.App.WorkingDirectory, bin)
	// filename, search in the app's $PATH
	default:
		absRoot, err := filepath.Abs(p.Root)
		if err != nil {
			return "", errwrap.Wrap(errors.New("could not get pod's root absolute path"), err)
		}
		appRootfs := common.AppRootfsPath(absRoot, ra.Name)
		appPathDirs := appSearchPaths(p, ra.App.WorkingDirectory, *ra.App)
		appPath := strings.Join(appPathDirs, ":")

		binPath, err = lookupPathInsideApp(bin, appPath, appRootfs, ra.App.WorkingDirectory)
		if err != nil {
			return "", errwrap.Wrap(fmt.Errorf("error looking up %q", bin), err)
		}
	}

	return binPath, nil
}

func copyResolv(p *stage1commontypes.Pod) error {
	ra := p.Manifest.Apps[0]

	stage1Rootfs := common.Stage1RootfsPath(p.Root)
	resolvPath := filepath.Join(stage1Rootfs, "etc", "rkt-resolv.conf")

	appRootfs := common.AppRootfsPath(p.Root, ra.Name)
	targetEtc := filepath.Join(appRootfs, "etc")
	targetResolvPath := filepath.Join(targetEtc, "resolv.conf")

	_, err := os.Stat(resolvPath)
	switch {
	case os.IsNotExist(err):
		return nil
	case err != nil:
		return err
	}

	_, err = os.Stat(targetResolvPath)
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	return fileutil.CopyRegularFile(resolvPath, targetResolvPath)
}

func mountSharedVolumes(root string, p *stage1commontypes.Pod, ra *schema.RuntimeApp) error {
	appName := ra.Name

	sharedVolPath := common.SharedVolumesPath(root)
	if err := os.MkdirAll(sharedVolPath, stage1initcommon.SharedVolPerm); err != nil {
		return errwrap.Wrap(errors.New("could not create shared volumes directory"), err)
	}
	if err := os.Chmod(sharedVolPath, stage1initcommon.SharedVolPerm); err != nil {
		return errwrap.Wrap(fmt.Errorf("could not change permissions of %q", sharedVolPath), err)
	}

	imageManifest := p.Images[appName.String()]
	mounts, err := stage1initcommon.GenerateMounts(ra, p.Manifest.Volumes, stage1initcommon.ConvertedFromDocker(imageManifest))
	if err != nil {
		return err
	}
	for _, m := range mounts {
		absRoot, err := filepath.Abs(p.Root) // Absolute path to the pod's rootfs.
		if err != nil {
			return errwrap.Wrap(errors.New("could not get pod's root absolute path"), err)
		}

		absAppRootfs := common.AppRootfsPath(absRoot, appName)
		if err != nil {
			return fmt.Errorf(`could not evaluate absolute path for application rootfs in app: %v`, appName)
		}

		mntPath, err := stage1initcommon.EvaluateSymlinksInsideApp(absAppRootfs, m.Mount.Path)
		if err != nil {
			return errwrap.Wrap(fmt.Errorf("could not evaluate path %v", m.Mount.Path), err)
		}
		absDestination := filepath.Join(absAppRootfs, mntPath)
		shPath := filepath.Join(sharedVolPath, m.Volume.Name.String())
		if err := stage1initcommon.PrepareMountpoints(shPath, absDestination, &m.Volume, m.DockerImplicit); err != nil {
			return err
		}

		var source string
		switch m.Volume.Kind {
		case "host":
			source = m.Volume.Source
		case "empty":
			source = filepath.Join(common.SharedVolumesPath(root), m.Volume.Name.String())
		default:
			return fmt.Errorf(`invalid volume kind %q. Must be one of "host" or "empty"`, m.Volume.Kind)
		}
		if cleanedSource, err := filepath.EvalSymlinks(source); err != nil {
			return errwrap.Wrap(fmt.Errorf("could not resolve symlink for source: %v", source), err)
		} else if err := ensureDestinationExists(cleanedSource, absDestination); err != nil {
			return errwrap.Wrap(fmt.Errorf("could not create destination mount point: %v", absDestination), err)
		} else if err := doBindMount(cleanedSource, absDestination, m.ReadOnly, m.Volume.Recursive); err != nil {
			return errwrap.Wrap(fmt.Errorf("could not bind mount path %v (s: %v, d: %v)", m.Mount.Path, source, absDestination), err)
		}
	}
	return nil
}

func AppAddMounts(p *stage1commontypes.Pod, ra *schema.RuntimeApp, enterCmd []string) error {
	sharedVolPath, err := common.CreateSharedVolumesPath(p.Root)
	if err != nil {
		return err
	}

	vols := make(map[types.ACName]types.Volume)
	for _, v := range p.Manifest.Volumes {
		vols[v.Name] = v
	}

	imageManifest := p.Images[ra.Name.String()]

	mounts, err := GenerateMounts(ra, p.Manifest.Volumes, ConvertedFromDocker(imageManifest))
	if err != nil {
		log.FatalE("Could not generate mounts", err)
		os.Exit(254)
	}

	absRoot, err := filepath.Abs(p.Root)
	if err != nil {
		log.FatalE("could not determine pod's absolute path", err)
	}

	appRootfs := common.AppRootfsPath(absRoot, ra.Name)

	// This logic is mostly copied from appToNspawnArgs
	// TODO(cdc): deduplicate
	for _, m := range mounts {
		shPath := filepath.Join(sharedVolPath, m.Volume.Name.String())

		// Evaluate symlinks within the app's rootfs - otherwise absolute
		// symlinks will be wrong.
		mntPath, err := EvaluateSymlinksInsideApp(appRootfs, m.Mount.Path)
		if err != nil {
			log.Fatalf("Could not evaluate path %v: %v", m.Mount.Path, err)
		}
		mntAbsPath := filepath.Join(appRootfs, mntPath)
		// Create the stage1 destination
		if err := PrepareMountpoints(shPath, mntAbsPath, &m.Volume, m.DockerImplicit); err != nil {
			log.FatalE("could not prepare mountpoint", err)
		}
		err = AppAddOneMount(p, ra, m.Source(absRoot), m.Mount.Path, m.ReadOnly, enterCmd)
		if err != nil {
			log.FatalE("Unable to setup app mounts", err)
		}
	}
	return nil
}

func mountSharedVolumes(p *stage1commontypes.Pod, ra *schema.RuntimeApp) error {
	appName := ra.Name

	sharedVolPath, err := common.CreateSharedVolumesPath(p.Root)
	if err != nil {
		return err
	}

	imageManifest := p.Images[appName.String()]
	mounts, err := stage1initcommon.GenerateMounts(ra, p.Manifest.Volumes, stage1initcommon.ConvertedFromDocker(imageManifest))
	if err != nil {
		return err
	}
	for _, m := range mounts {
		absRoot, err := filepath.Abs(p.Root) // Absolute path to the pod's rootfs.
		if err != nil {
			return errwrap.Wrap(errors.New("could not get pod's root absolute path"), err)
		}

		absAppRootfs := common.AppRootfsPath(absRoot, appName)
		if err != nil {
			return fmt.Errorf(`could not evaluate absolute path for application rootfs in app: %v`, appName)
		}

		mntPath, err := stage1initcommon.EvaluateSymlinksInsideApp(absAppRootfs, m.Mount.Path)
		if err != nil {
			return errwrap.Wrap(fmt.Errorf("could not evaluate path %v", m.Mount.Path), err)
		}
		absDestination := filepath.Join(absAppRootfs, mntPath)
		shPath := filepath.Join(sharedVolPath, m.Volume.Name.String())
		if err := stage1initcommon.PrepareMountpoints(shPath, absDestination, &m.Volume, m.DockerImplicit); err != nil {
			return err
		}

		source := m.Source(p.Root)
		if cleanedSource, err := filepath.EvalSymlinks(source); err != nil {
			return errwrap.Wrap(fmt.Errorf("could not resolve symlink for source: %v", source), err)
		} else if err := ensureDestinationExists(cleanedSource, absDestination); err != nil {
			return errwrap.Wrap(fmt.Errorf("could not create destination mount point: %v", absDestination), err)
		} else if err := doBindMount(cleanedSource, absDestination, m.ReadOnly, m.Volume.Recursive); err != nil {
			return errwrap.Wrap(fmt.Errorf("could not bind mount path %v (s: %v, d: %v)", m.Mount.Path, source, absDestination), err)
		}
	}
	return nil
}

// parseUserGroup parses the User and Group fields of an App and returns its
// UID and GID.
// The User and Group fields accept several formats:
//   1. the hardcoded string "root"
//   2. a path
//   3. a number
//   4. a name in reference to /etc/{group,passwd} in the image
// See https://github.com/appc/spec/blob/master/spec/aci.md#image-manifest-schema
func parseUserGroup(p *stage1commontypes.Pod, ra *schema.RuntimeApp) (int, int, error) {
	var uidResolver, gidResolver user.Resolver
	var uid, gid int
	var err error

	root := common.AppRootfsPath(p.Root, ra.Name)

	uidResolver, err = user.NumericIDs(ra.App.User)
	if err != nil {
		uidResolver, err = user.IDsFromStat(root, ra.App.User, &p.UidRange)
	}

	if err != nil {
		uidResolver, err = user.IDsFromEtc(root, ra.App.User, "")
	}

	if err != nil { // give up
		return -1, -1, errwrap.Wrap(fmt.Errorf("invalid user %q", ra.App.User), err)
	}

	if uid, _, err = uidResolver.IDs(); err != nil {
		return -1, -1, errwrap.Wrap(fmt.Errorf("failed to configure user %q", ra.App.User), err)
	}

	gidResolver, err = user.NumericIDs(ra.App.Group)
	if err != nil {
		gidResolver, err = user.IDsFromStat(root, ra.App.Group, &p.UidRange)
	}

	if err != nil {
		gidResolver, err = user.IDsFromEtc(root, "", ra.App.Group)
	}

	if err != nil { // give up
		return -1, -1, errwrap.Wrap(fmt.Errorf("invalid group %q", ra.App.Group), err)
	}

	if _, gid, err = gidResolver.IDs(); err != nil {
		return -1, -1, errwrap.Wrap(fmt.Errorf("failed to configure group %q", ra.App.Group), err)
	}

	return uid, gid, nil
}

// appToNspawnArgs transforms the given app manifest, with the given associated
// app name, into a subset of applicable systemd-nspawn argument
func appToNspawnArgs(p *stage1commontypes.Pod, ra *schema.RuntimeApp) ([]string, error) {
	var args []string
	appName := ra.Name
	app := ra.App

	sharedVolPath := common.SharedVolumesPath(p.Root)
	if err := os.MkdirAll(sharedVolPath, sharedVolPerm); err != nil {
		return nil, errwrap.Wrap(errors.New("could not create shared volumes directory"), err)
	}
	if err := os.Chmod(sharedVolPath, sharedVolPerm); err != nil {
		return nil, errwrap.Wrap(fmt.Errorf("could not change permissions of %q", sharedVolPath), err)
	}

	vols := make(map[types.ACName]types.Volume)
	for _, v := range p.Manifest.Volumes {
		vols[v.Name] = v
	}

	mounts := GenerateMounts(ra, vols)
	for _, m := range mounts {
		vol := vols[m.Volume]

		if vol.Kind == "empty" {
			p := filepath.Join(sharedVolPath, vol.Name.String())
			if err := os.MkdirAll(p, sharedVolPerm); err != nil {
				return nil, errwrap.Wrap(fmt.Errorf("could not create shared volume %q", vol.Name), err)
			}
			if err := os.Chown(p, *vol.UID, *vol.GID); err != nil {
				return nil, errwrap.Wrap(fmt.Errorf("could not change owner of %q", p), err)
			}
			mod, err := strconv.ParseUint(*vol.Mode, 8, 32)
			if err != nil {
				return nil, errwrap.Wrap(fmt.Errorf("invalid mode %q for volume %q", *vol.Mode, vol.Name), err)
			}
			if err := os.Chmod(p, os.FileMode(mod)); err != nil {
				return nil, errwrap.Wrap(fmt.Errorf("could not change permissions of %q", p), err)
			}
		}

		opt := make([]string, 4)

		if IsMountReadOnly(vol, app.MountPoints) {
			opt[0] = "--bind-ro="
		} else {
			opt[0] = "--bind="
		}

		absRoot, err := filepath.Abs(p.Root) // Absolute path to the pod's rootfs.
		if err != nil {
			return nil, errwrap.Wrap(errors.New("could not get pod's root absolute path"), err)
		}

		switch vol.Kind {
		case "host":
			opt[1] = vol.Source
		case "empty":
			opt[1] = filepath.Join(common.SharedVolumesPath(absRoot), vol.Name.String())
		default:
			return nil, fmt.Errorf(`invalid volume kind %q. Must be one of "host" or "empty"`, vol.Kind)
		}
		opt[2] = ":"

		appRootfs := common.AppRootfsPath(absRoot, appName)
		mntPath, err := evaluateAppMountPath(appRootfs, m.Path)
		if err != nil {
			return nil, errwrap.Wrap(fmt.Errorf("could not evaluate path %v", m.Path), err)
		}

		opt[3] = filepath.Join(common.RelAppRootfsPath(appName), mntPath)

		args = append(args, strings.Join(opt, ""))
	}

	for _, i := range app.Isolators {
		switch v := i.Value().(type) {
		case types.LinuxCapabilitiesSet:
			var caps []string
			// TODO: cleanup the API on LinuxCapabilitiesSet to give strings easily.
			for _, c := range v.Set() {
				caps = append(caps, string(c))
			}
			if i.Name == types.LinuxCapabilitiesRetainSetName {
				capList := strings.Join(caps, ",")
				args = append(args, "--capability="+capList)
			}
		}
	}

	return args, nil
}

//.........这里部分代码省略.........
	 * 'default' - do nothing (we would respect CNI if fly had networking)
	 * 'none' - do nothing
	 */
	switch p.ResolvConfMode {
	case "host":
		effectiveMounts = append(effectiveMounts,
			flyMount{"/etc/resolv.conf", rfs, "/etc/resolv.conf", "none", syscall.MS_BIND | syscall.MS_RDONLY})
	case "stage0":
		if err := copyResolv(p); err != nil {
			log.PrintE("can't copy /etc/resolv.conf", err)
			return 254
		}
	}

	/*
	 * /etc/hosts: three modes:
	 * 'host' - bind-mount hosts's file
	 * 'stage0' - bind mount the file created by stage1
	 * 'default' - create a stub /etc/hosts if needed
	 */

	switch p.EtcHostsMode {
	case "host":
		effectiveMounts = append(effectiveMounts,
			flyMount{"/etc/hosts", rfs, "/etc/hosts", "none", syscall.MS_BIND | syscall.MS_RDONLY})
	case "stage0":
		effectiveMounts = append(effectiveMounts, flyMount{
			filepath.Join(common.Stage1RootfsPath(p.Root), "etc", "rkt-hosts"),
			rfs,
			"/etc/hosts",
			"none",
			syscall.MS_BIND | syscall.MS_RDONLY})
	case "default":
		stage2HostsPath := filepath.Join(common.AppRootfsPath(p.Root, ra.Name), "etc", "hosts")
		if _, err := os.Stat(stage2HostsPath); err != nil && os.IsNotExist(err) {
			fallbackHosts := []byte("127.0.0.1 localhost localdomain\n")
			ioutil.WriteFile(stage2HostsPath, fallbackHosts, 0644)
		}
	}

	for _, mount := range effectiveMounts {
		diag.Printf("Processing %+v", mount)

		var (
			err            error
			hostPathInfo   os.FileInfo
			targetPathInfo os.FileInfo
		)

		if strings.HasPrefix(mount.HostPath, "/") {
			if hostPathInfo, err = os.Stat(mount.HostPath); err != nil {
				log.PrintE(fmt.Sprintf("stat of host path %s", mount.HostPath), err)
				return 254
			}
		} else {
			hostPathInfo = nil
		}

		absTargetPath := filepath.Join(mount.TargetPrefixPath, mount.RelTargetPath)
		if targetPathInfo, err = os.Stat(absTargetPath); err != nil && !os.IsNotExist(err) {
			log.PrintE(fmt.Sprintf("stat of target path %s", absTargetPath), err)
			return 254
		}

		switch {
		case (mount.Flags & syscall.MS_REMOUNT) != 0:

func (uw *UnitWriter) AppUnit(
	ra *schema.RuntimeApp, binPath, privateUsers string, insecureOptions Stage1InsecureOptions,
	opts ...*unit.UnitOption,
) {
	if uw.err != nil {
		return
	}

	flavor, systemdVersion, err := GetFlavor(uw.p)
	if err != nil {
		uw.err = errwrap.Wrap(errors.New("unable to determine stage1 flavor"), err)
		return
	}

	app := ra.App
	appName := ra.Name
	imgName := uw.p.AppNameToImageName(appName)

	if len(app.Exec) == 0 {
		uw.err = fmt.Errorf(`image %q has an empty "exec" (try --exec=BINARY)`, imgName)
		return
	}

	env := app.Environment

	env.Set("AC_APP_NAME", appName.String())
	if uw.p.MetadataServiceURL != "" {
		env.Set("AC_METADATA_URL", uw.p.MetadataServiceURL)
	}

	envFilePath := EnvFilePath(uw.p.Root, appName)

	uidRange := user.NewBlankUidRange()
	if err := uidRange.Deserialize([]byte(privateUsers)); err != nil {
		uw.err = err
		return
	}

	if err := common.WriteEnvFile(env, uidRange, envFilePath); err != nil {
		uw.err = errwrap.Wrap(errors.New("unable to write environment file for systemd"), err)
		return
	}

	u, g, err := parseUserGroup(uw.p, ra, uidRange)
	if err != nil {
		uw.err = err
		return
	}

	if err := generateSysusers(uw.p, ra, u, g, uidRange); err != nil {
		uw.err = errwrap.Wrap(errors.New("unable to generate sysusers"), err)
		return
	}

	var supplementaryGroups []string
	for _, g := range app.SupplementaryGIDs {
		supplementaryGroups = append(supplementaryGroups, strconv.Itoa(g))
	}

	capabilitiesStr, err := getAppCapabilities(app.Isolators)
	if err != nil {
		uw.err = err
		return
	}

	execStart := append([]string{binPath}, app.Exec[1:]...)
	execStartString := quoteExec(execStart)
	opts = append(opts, []*unit.UnitOption{
		unit.NewUnitOption("Unit", "Description", fmt.Sprintf("Application=%v Image=%v", appName, imgName)),
		unit.NewUnitOption("Unit", "DefaultDependencies", "false"),
		unit.NewUnitOption("Unit", "Wants", fmt.Sprintf("reaper-%s.service", appName)),
		unit.NewUnitOption("Service", "Restart", "no"),
		unit.NewUnitOption("Service", "ExecStart", execStartString),
		unit.NewUnitOption("Service", "RootDirectory", common.RelAppRootfsPath(appName)),
		// MountFlags=shared creates a new mount namespace and (as unintuitive
		// as it might seem) makes sure the mount is slave+shared.
		unit.NewUnitOption("Service", "MountFlags", "shared"),
		unit.NewUnitOption("Service", "WorkingDirectory", app.WorkingDirectory),
		unit.NewUnitOption("Service", "EnvironmentFile", RelEnvFilePath(appName)),
		unit.NewUnitOption("Service", "User", strconv.Itoa(u)),
		unit.NewUnitOption("Service", "Group", strconv.Itoa(g)),

		// This helps working around a race
		// (https://github.com/systemd/systemd/issues/2913) that causes the
		// systemd unit name not getting written to the journal if the unit is
		// short-lived and runs as non-root.
		unit.NewUnitOption("Service", "SyslogIdentifier", appName.String()),
	}...)

	if len(supplementaryGroups) > 0 {
		opts = appendOptionsList(opts, "Service", "SupplementaryGroups", "", supplementaryGroups)
	}

	if supportsNotify(uw.p, appName.String()) {
		opts = append(opts, unit.NewUnitOption("Service", "Type", "notify"))
	}

	if !insecureOptions.DisableCapabilities {
		opts = append(opts, unit.NewUnitOption("Service", "CapabilityBoundingSet", strings.Join(capabilitiesStr, " ")))
	}
//.........这里部分代码省略.........

// TODO(iaguis): RmConfig?
func RmApp(dir string, uuid *types.UUID, usesOverlay bool, appName *types.ACName, podPID int) error {
	p, err := stage1types.LoadPod(dir, uuid)
	if err != nil {
		return errwrap.Wrap(errors.New("error loading pod manifest"), err)
	}

	pm := p.Manifest

	var mutable bool
	ms, ok := pm.Annotations.Get("coreos.com/rkt/stage1/mutable")
	if ok {
		mutable, err = strconv.ParseBool(ms)
		if err != nil {
			return errwrap.Wrap(errors.New("error parsing mutable annotation"), err)
		}
	}

	if !mutable {
		return errors.New("immutable pod: cannot remove application")
	}

	app := pm.Apps.Get(*appName)
	if app == nil {
		return fmt.Errorf("error: nonexistent app %q", *appName)
	}

	treeStoreID, err := ioutil.ReadFile(common.AppTreeStoreIDPath(dir, *appName))
	if err != nil {
		return err
	}

	eep, err := getStage1Entrypoint(dir, enterEntrypoint)
	if err != nil {
		return errwrap.Wrap(errors.New("error determining 'enter' entrypoint"), err)
	}

	args := []string{
		uuid.String(),
		appName.String(),
		filepath.Join(common.Stage1RootfsPath(dir), eep),
		strconv.Itoa(podPID),
	}

	if err := callEntrypoint(dir, appStopEntrypoint, args); err != nil {
		return err
	}

	if err := callEntrypoint(dir, appRmEntrypoint, args); err != nil {
		return err
	}

	appInfoDir := common.AppInfoPath(dir, *appName)
	if err := os.RemoveAll(appInfoDir); err != nil {
		return errwrap.Wrap(errors.New("error removing app info directory"), err)
	}

	if usesOverlay {
		appRootfs := common.AppRootfsPath(dir, *appName)
		if err := syscall.Unmount(appRootfs, 0); err != nil {
			return err
		}

		ts := filepath.Join(dir, "overlay", string(treeStoreID))
		if err := os.RemoveAll(ts); err != nil {
			return errwrap.Wrap(errors.New("error removing app info directory"), err)
		}
	}

	if err := os.RemoveAll(common.AppPath(dir, *appName)); err != nil {
		return err
	}

	appStatusPath := filepath.Join(common.Stage1RootfsPath(dir), "rkt", "status", appName.String())
	if err := os.Remove(appStatusPath); err != nil && !os.IsNotExist(err) {
		return err
	}

	envPath := filepath.Join(common.Stage1RootfsPath(dir), "rkt", "env", appName.String())
	if err := os.Remove(envPath); err != nil && !os.IsNotExist(err) {
		return err
	}

	removeAppFromPodManifest(pm, appName)

	if err := updatePodManifest(dir, pm); err != nil {
		return err
	}

	return nil
}

// appToNspawnArgs transforms the given app manifest, with the given associated
// app name, into a subset of applicable systemd-nspawn argument
func appToNspawnArgs(p *stage1commontypes.Pod, ra *schema.RuntimeApp, insecureOptions Stage1InsecureOptions) ([]string, error) {
	var args []string
	appName := ra.Name
	app := ra.App

	sharedVolPath := common.SharedVolumesPath(p.Root)
	if err := os.MkdirAll(sharedVolPath, SharedVolPerm); err != nil {
		return nil, errwrap.Wrap(errors.New("could not create shared volumes directory"), err)
	}
	if err := os.Chmod(sharedVolPath, SharedVolPerm); err != nil {
		return nil, errwrap.Wrap(fmt.Errorf("could not change permissions of %q", sharedVolPath), err)
	}

	vols := make(map[types.ACName]types.Volume)
	for _, v := range p.Manifest.Volumes {
		vols[v.Name] = v
	}

	imageManifest := p.Images[appName.String()]
	mounts, err := GenerateMounts(ra, p.Manifest.Volumes, ConvertedFromDocker(imageManifest))
	if err != nil {
		return nil, errwrap.Wrap(fmt.Errorf("could not generate app %q mounts", appName), err)
	}
	for _, m := range mounts {

		shPath := filepath.Join(sharedVolPath, m.Volume.Name.String())

		absRoot, err := filepath.Abs(p.Root) // Absolute path to the pod's rootfs.
		if err != nil {
			return nil, errwrap.Wrap(errors.New("could not get pod's root absolute path"), err)
		}

		appRootfs := common.AppRootfsPath(absRoot, appName)

		// TODO(yifan): This is a temporary fix for systemd-nspawn not handling symlink mounts well.
		// Could be removed when https://github.com/systemd/systemd/issues/2860 is resolved, and systemd
		// version is bumped.
		mntPath, err := EvaluateSymlinksInsideApp(appRootfs, m.Mount.Path)
		if err != nil {
			return nil, errwrap.Wrap(fmt.Errorf("could not evaluate path %v", m.Mount.Path), err)
		}
		mntAbsPath := filepath.Join(appRootfs, mntPath)

		if err := PrepareMountpoints(shPath, mntAbsPath, &m.Volume, m.DockerImplicit); err != nil {
			return nil, err
		}

		opt := make([]string, 6)

		if m.ReadOnly {
			opt[0] = "--bind-ro="
		} else {
			opt[0] = "--bind="
		}

		switch m.Volume.Kind {
		case "host":
			opt[1] = m.Volume.Source
		case "empty":
			opt[1] = filepath.Join(common.SharedVolumesPath(absRoot), m.Volume.Name.String())
		default:
			return nil, fmt.Errorf(`invalid volume kind %q. Must be one of "host" or "empty"`, m.Volume.Kind)
		}
		opt[2] = ":"
		opt[3] = filepath.Join(common.RelAppRootfsPath(appName), mntPath)
		opt[4] = ":"

		// If Recursive is not set, default to recursive.
		recursive := true
		if m.Volume.Recursive != nil {
			recursive = *m.Volume.Recursive
		}

		// rbind/norbind options exist since systemd-nspawn v226
		if recursive {
			opt[5] = "rbind"
		} else {
			opt[5] = "norbind"
		}
		args = append(args, strings.Join(opt, ""))
	}

	if !insecureOptions.DisableCapabilities {
		capabilitiesStr, err := getAppCapabilities(app.Isolators)
		if err != nil {
			return nil, err
		}
		capList := strings.Join(capabilitiesStr, ",")
		args = append(args, "--capability="+capList)
	}

	return args, nil
}

func RmApp(cfg RmConfig) error {
	pod, err := pkgPod.PodFromUUIDString(cfg.DataDir, cfg.UUID.String())
	if err != nil {
		return errwrap.Wrap(errors.New("error loading pod"), err)
	}
	defer pod.Close()

	debug("locking sandbox manifest")
	if err := pod.ExclusiveLockManifest(); err != nil {
		return errwrap.Wrap(errors.New("failed to lock sandbox manifest"), err)
	}
	defer pod.UnlockManifest()

	pm, err := pod.SandboxManifest()
	if err != nil {
		return errwrap.Wrap(errors.New("cannot remove application, sandbox validation failed"), err)
	}

	app := pm.Apps.Get(*cfg.AppName)
	if app == nil {
		return fmt.Errorf("error: nonexistent app %q", *cfg.AppName)
	}

	if cfg.PodPID > 0 {
		// Call app-stop and app-rm entrypoint only if the pod is still running.
		// Otherwise, there's not much we can do about it except unmounting/removing
		// the file system.
		args := []string{
			fmt.Sprintf("--debug=%t", cfg.Debug),
			fmt.Sprintf("--app=%s", cfg.AppName),
		}

		ce := CrossingEntrypoint{
			PodPath:        cfg.PodPath,
			PodPID:         cfg.PodPID,
			AppName:        cfg.AppName.String(),
			EntrypointName: appStopEntrypoint,
			EntrypointArgs: args,
			Interactive:    false,
		}
		if err := ce.Run(); err != nil {
			status, err := common.GetExitStatus(err)
			// ignore nonexistent units failing to stop. Exit status 5
			// comes from systemctl and means the unit doesn't exist
			if err != nil {
				return err
			} else if status != 5 {
				return fmt.Errorf("exit status %d", status)
			}
		}

		ce.EntrypointName = appRmEntrypoint
		if err := ce.Run(); err != nil {
			return err
		}
	}

	if cfg.UsesOverlay {
		treeStoreID, err := ioutil.ReadFile(common.AppTreeStoreIDPath(cfg.PodPath, *cfg.AppName))
		if err != nil {
			return err
		}

		appRootfs := common.AppRootfsPath(cfg.PodPath, *cfg.AppName)
		if err := syscall.Unmount(appRootfs, 0); err != nil {
			return err
		}

		ts := filepath.Join(cfg.PodPath, "overlay", string(treeStoreID))
		if err := os.RemoveAll(ts); err != nil {
			return errwrap.Wrap(errors.New("error removing app info directory"), err)
		}
	}

	appInfoDir := common.AppInfoPath(cfg.PodPath, *cfg.AppName)
	if err := os.RemoveAll(appInfoDir); err != nil {
		return errwrap.Wrap(errors.New("error removing app info directory"), err)
	}

	if err := os.RemoveAll(common.AppPath(cfg.PodPath, *cfg.AppName)); err != nil {
		return err
	}

	appStatusPath := filepath.Join(common.Stage1RootfsPath(cfg.PodPath), "rkt", "status", cfg.AppName.String())
	if err := os.Remove(appStatusPath); err != nil && !os.IsNotExist(err) {
		return err
	}

	envPath := filepath.Join(common.Stage1RootfsPath(cfg.PodPath), "rkt", "env", cfg.AppName.String())
	if err := os.Remove(envPath); err != nil && !os.IsNotExist(err) {
		return err
	}

	for i, app := range pm.Apps {
		if app.Name == *cfg.AppName {
			pm.Apps = append(pm.Apps[:i], pm.Apps[i+1:]...)
			break
		}
	}

//.........这里部分代码省略.........

// appToSystemd transforms the provided RuntimeApp+ImageManifest into systemd units
func appToSystemd(p *stage1commontypes.Pod, ra *schema.RuntimeApp, interactive bool, flavor string, privateUsers string) error {
	app := ra.App
	appName := ra.Name
	imgName := p.AppNameToImageName(appName)

	if len(app.Exec) == 0 {
		return fmt.Errorf(`image %q has an empty "exec" (try --exec=BINARY)`, imgName)
	}

	workDir := "/"
	if app.WorkingDirectory != "" {
		workDir = app.WorkingDirectory
	}

	env := app.Environment

	env.Set("AC_APP_NAME", appName.String())
	if p.MetadataServiceURL != "" {
		env.Set("AC_METADATA_URL", p.MetadataServiceURL)
	}

	envFilePath := EnvFilePath(p.Root, appName)

	uidRange := user.NewBlankUidRange()
	if err := uidRange.Deserialize([]byte(privateUsers)); err != nil {
		return err
	}

	if err := writeEnvFile(p, env, appName, uidRange, '\n', envFilePath); err != nil {
		return errwrap.Wrap(errors.New("unable to write environment file for systemd"), err)
	}

	u, g, err := parseUserGroup(p, ra, uidRange)
	if err != nil {
		return err
	}

	if err := generateSysusers(p, ra, u, g, uidRange); err != nil {
		return errwrap.Wrap(errors.New("unable to generate sysusers"), err)
	}

	binPath, err := findBinPath(p, appName, *app, workDir, app.Exec[0])
	if err != nil {
		return err
	}

	var supplementaryGroups []string
	for _, g := range app.SupplementaryGIDs {
		supplementaryGroups = append(supplementaryGroups, strconv.Itoa(g))
	}

	capabilitiesStr, err := getAppCapabilities(app.Isolators)
	if err != nil {
		return err
	}

	noNewPrivileges := getAppNoNewPrivileges(app.Isolators)

	execStart := append([]string{binPath}, app.Exec[1:]...)
	execStartString := quoteExec(execStart)
	opts := []*unit.UnitOption{
		unit.NewUnitOption("Unit", "Description", fmt.Sprintf("Application=%v Image=%v", appName, imgName)),
		unit.NewUnitOption("Unit", "DefaultDependencies", "false"),
		unit.NewUnitOption("Unit", "Wants", fmt.Sprintf("reaper-%s.service", appName)),
		unit.NewUnitOption("Service", "Restart", "no"),
		unit.NewUnitOption("Service", "ExecStart", execStartString),
		unit.NewUnitOption("Service", "RootDirectory", common.RelAppRootfsPath(appName)),
		// MountFlags=shared creates a new mount namespace and (as unintuitive
		// as it might seem) makes sure the mount is slave+shared.
		unit.NewUnitOption("Service", "MountFlags", "shared"),
		unit.NewUnitOption("Service", "WorkingDirectory", workDir),
		unit.NewUnitOption("Service", "EnvironmentFile", RelEnvFilePath(appName)),
		unit.NewUnitOption("Service", "User", strconv.Itoa(u)),
		unit.NewUnitOption("Service", "Group", strconv.Itoa(g)),
		unit.NewUnitOption("Service", "SupplementaryGroups", strings.Join(supplementaryGroups, " ")),
		unit.NewUnitOption("Service", "CapabilityBoundingSet", strings.Join(capabilitiesStr, " ")),
		unit.NewUnitOption("Service", "NoNewPrivileges", strconv.FormatBool(noNewPrivileges)),
		// This helps working around a race
		// (https://github.com/systemd/systemd/issues/2913) that causes the
		// systemd unit name not getting written to the journal if the unit is
		// short-lived and runs as non-root.
		unit.NewUnitOption("Service", "SyslogIdentifier", appName.String()),
	}

	// Restrict access to sensitive paths (eg. procfs)
	opts = protectSystemFiles(opts, appName)

	if ra.ReadOnlyRootFS {
		opts = append(opts, unit.NewUnitOption("Service", "ReadOnlyDirectories", common.RelAppRootfsPath(appName)))
	}

	// TODO(tmrts): Extract this logic into a utility function.
	vols := make(map[types.ACName]types.Volume)
	for _, v := range p.Manifest.Volumes {
		vols[v.Name] = v
	}

	absRoot, err := filepath.Abs(p.Root) // Absolute path to the pod's rootfs.
	if err != nil {
//.........这里部分代码省略.........

func RmApp(cfg RmConfig) error {
	pod, err := pkgPod.PodFromUUIDString(cfg.DataDir, cfg.UUID.String())
	if err != nil {
		return errwrap.Wrap(errors.New("error loading pod"), err)
	}
	defer pod.Close()

	debug("locking pod manifest")
	if err := pod.ExclusiveManifestLock(); err != nil {
		return errwrap.Wrap(errors.New("failed to lock pod manifest"), err)
	}
	defer pod.ManifestUnlock()

	_, pm, err := pod.PodManifest()
	if err != nil {
		return errwrap.Wrap(errors.New("error loading pod manifest"), err)
	}

	var mutable bool
	ms, ok := pm.Annotations.Get("coreos.com/rkt/stage1/mutable")
	if ok {
		mutable, err = strconv.ParseBool(ms)
		if err != nil {
			return errwrap.Wrap(errors.New("error parsing mutable annotation"), err)
		}
	}

	if !mutable {
		return errors.New("immutable pod: cannot remove application")
	}

	app := pm.Apps.Get(*cfg.AppName)
	if app == nil {
		return fmt.Errorf("error: nonexistent app %q", *cfg.AppName)
	}

	if cfg.PodPID > 0 {
		// Call app-stop and app-rm entrypoint only if the pod is still running.
		// Otherwise, there's not much we can do about it except unmounting/removing
		// the file system.
		args := []string{
			fmt.Sprintf("--app=%s", cfg.AppName),
		}

		ce := CrossingEntrypoint{
			PodPath:        cfg.PodPath,
			PodPID:         cfg.PodPID,
			AppName:        cfg.AppName.String(),
			EntrypointName: appStopEntrypoint,
			EntrypointArgs: args,
			Interactive:    false,
		}
		if err := ce.Run(); err != nil {
			status, err := common.GetExitStatus(err)
			// ignore nonexistent units failing to stop. Exit status 5
			// comes from systemctl and means the unit doesn't exist
			if err != nil {
				return err
			} else if status != 5 {
				return fmt.Errorf("exit status %d", status)
			}
		}

		ce.EntrypointName = appRmEntrypoint
		if err := ce.Run(); err != nil {
			return err
		}
	}

	if cfg.UsesOverlay {
		treeStoreID, err := ioutil.ReadFile(common.AppTreeStoreIDPath(cfg.PodPath, *cfg.AppName))
		if err != nil {
			return err
		}

		appRootfs := common.AppRootfsPath(cfg.PodPath, *cfg.AppName)
		if err := syscall.Unmount(appRootfs, 0); err != nil {
			return err
		}

		ts := filepath.Join(cfg.PodPath, "overlay", string(treeStoreID))
		if err := os.RemoveAll(ts); err != nil {
			return errwrap.Wrap(errors.New("error removing app info directory"), err)
		}
	}

	appInfoDir := common.AppInfoPath(cfg.PodPath, *cfg.AppName)
	if err := os.RemoveAll(appInfoDir); err != nil {
		return errwrap.Wrap(errors.New("error removing app info directory"), err)
	}

	if err := os.RemoveAll(common.AppPath(cfg.PodPath, *cfg.AppName)); err != nil {
		return err
	}

	appStatusPath := filepath.Join(common.Stage1RootfsPath(cfg.PodPath), "rkt", "status", cfg.AppName.String())
	if err := os.Remove(appStatusPath); err != nil && !os.IsNotExist(err) {
		return err
	}

//.........这里部分代码省略.........

func runExport(cmd *cobra.Command, args []string) (exit int) {
	if len(args) != 2 {
		cmd.Usage()
		return 254
	}

	outACI := args[1]
	ext := filepath.Ext(outACI)
	if ext != schema.ACIExtension {
		stderr.Printf("extension must be %s (given %s)", schema.ACIExtension, outACI)
		return 254
	}

	p, err := pkgPod.PodFromUUIDString(getDataDir(), args[0])
	if err != nil {
		stderr.PrintE("problem retrieving pod", err)
		return 254
	}
	defer p.Close()

	state := p.State()
	if state != pkgPod.Exited && state != pkgPod.ExitedGarbage {
		stderr.Print("pod is not exited. Only exited pods can be exported")
		return 254
	}

	app, err := getApp(p)
	if err != nil {
		stderr.PrintE("unable to find app", err)
		return 254
	}

	root := common.AppPath(p.Path(), app.Name)
	manifestPath := filepath.Join(common.AppInfoPath(p.Path(), app.Name), aci.ManifestFile)
	if p.UsesOverlay() {
		tmpDir := filepath.Join(getDataDir(), "tmp")
		if err := os.MkdirAll(tmpDir, common.DefaultRegularDirPerm); err != nil {
			stderr.PrintE("unable to create temp directory", err)
			return 254
		}
		podDir, err := ioutil.TempDir(tmpDir, fmt.Sprintf("rkt-export-%s", p.UUID))
		if err != nil {
			stderr.PrintE("unable to create export temp directory", err)
			return 254
		}
		defer func() {
			if err := os.RemoveAll(podDir); err != nil {
				stderr.PrintE("problem removing temp directory", err)
				exit = 1
			}
		}()
		mntDir := filepath.Join(podDir, "rootfs")
		if err := os.Mkdir(mntDir, common.DefaultRegularDirPerm); err != nil {
			stderr.PrintE("unable to create rootfs directory inside temp directory", err)
			return 254
		}

		if err := mountOverlay(p, app, mntDir); err != nil {
			stderr.PrintE(fmt.Sprintf("couldn't mount directory at %s", mntDir), err)
			return 254
		}
		defer func() {
			if err := syscall.Unmount(mntDir, 0); err != nil {
				stderr.PrintE(fmt.Sprintf("error unmounting directory %s", mntDir), err)
				exit = 1
			}
		}()
		root = podDir
	} else {
		// trailing filepath separator so we don't match the appRootfs path
		appRootfs := common.AppRootfsPath(p.Path(), app.Name) + string(filepath.Separator)
		mnts, err := mountinfo.ParseMounts(0)
		if err != nil {
			stderr.PrintE("error parsing mountpoints", err)
			return 254
		}
		mnts = mnts.Filter(mountinfo.HasPrefix(appRootfs))
		if len(mnts) > 0 {
			stderr.Printf("pod has remaining mountpoints. Only pods using overlayfs or with no mountpoints can be exported")
			return 254
		}
	}

	// Check for user namespace (--private-user), if in use get uidRange
	var uidRange *user.UidRange
	privUserFile := filepath.Join(p.Path(), common.PrivateUsersPreparedFilename)
	privUserContent, err := ioutil.ReadFile(privUserFile)
	if err == nil {
		uidRange = user.NewBlankUidRange()
		// The file was found, save uid & gid shift and count
		if err := uidRange.Deserialize(privUserContent); err != nil {
			stderr.PrintE(fmt.Sprintf("problem deserializing the content of %s", common.PrivateUsersPreparedFilename), err)
			return 254
		}
	}

	if err = buildAci(root, manifestPath, outACI, uidRange); err != nil {
		stderr.PrintE("error building aci", err)
		return 254
	}
//.........这里部分代码省略.........