func createNodes(ctx context.Context, sess *session.Session, pool *object.ResourcePool, node *Node, base string) error {
	log.Debugf("create node %+v", node)
	if node == nil {
		return nil
	}
	spec := simulator.NewResourceConfigSpec()
	node.Name = fmt.Sprintf("%s-%s", base, node.Name)
	switch node.Kind {
	case rpNode:
		child, err := pool.Create(ctx, node.Name, spec)
		if err != nil {
			return err
		}
		for _, childNode := range node.Children {
			return createNodes(ctx, sess, child, childNode, base)
		}
	case vappNode:
		confSpec := types.VAppConfigSpec{
			VmConfigSpec: types.VmConfigSpec{},
		}
		vapp, err := pool.CreateVApp(ctx, node.Name, spec, confSpec, nil)
		if err != nil {
			return err
		}
		config := types.VirtualMachineConfigSpec{
			Name:    node.Name,
			GuestId: string(types.VirtualMachineGuestOsIdentifierOtherGuest),
			Files: &types.VirtualMachineFileInfo{
				VmPathName: fmt.Sprintf("[LocalDS_0] %s", node.Name),
			},
		}
		if _, err = tasks.WaitForResult(ctx, func(ctx context.Context) (tasks.Task, error) {
			return vapp.CreateChildVM_Task(ctx, config, nil)
		}); err != nil {
			return err
		}
	case vmNode:
		config := types.VirtualMachineConfigSpec{
			Name:    node.Name,
			GuestId: string(types.VirtualMachineGuestOsIdentifierOtherGuest),
			Files: &types.VirtualMachineFileInfo{
				VmPathName: fmt.Sprintf("[LocalDS_0] %s", node.Name),
			},
		}
		folder := sess.Folders(ctx).VmFolder
		if _, err := tasks.WaitForResult(ctx, func(ctx context.Context) (tasks.Task, error) {
			return folder.CreateVM(ctx, config, pool, nil)
		}); err != nil {
			return err
		}
	default:
		return nil
	}
	return nil
}

// WaitForResult is designed to handle VM invalid state error for any VM operations.
// It will call tasks.WaitForResult to retry if there is task in progress error.
func (vm *VirtualMachine) WaitForResult(ctx context.Context, f func(context.Context) (tasks.Task, error)) (*types.TaskInfo, error) {
	info, err := tasks.WaitForResult(ctx, f)
	if err == nil || !vm.needsFix(err) {
		return info, err
	}
	log.Debugf("Try to fix task failure %s", err)
	if nerr := vm.FixInvalidState(ctx); nerr != nil {
		log.Errorf("Failed to fix task failure: %s", nerr)
		return info, err
	}
	log.Debugf("Fixed")
	return tasks.WaitForResult(ctx, f)
}

func (d *Dispatcher) deleteVM(vm *vm.VirtualMachine, force bool) error {
	defer trace.End(trace.Begin(""))

	var err error
	power, err := vm.PowerState(d.ctx)
	if err != nil || power != types.VirtualMachinePowerStatePoweredOff {
		if err != nil {
			log.Warnf("Failed to get vm power status %s: %s", vm.Reference(), err)
		}
		if !force {
			if err != nil {
				return err
			}
			name, err := vm.Name(d.ctx)
			if err != nil {
				log.Errorf("VM name is not found, %s", err)
			}
			if name != "" {
				err = errors.Errorf("VM %s is powered on", name)
			} else {
				err = errors.Errorf("VM %s is powered on", vm.Reference())
			}
			return err
		}
		if _, err = tasks.WaitForResult(d.ctx, func(ctx context.Context) (tasks.ResultWaiter, error) {
			return vm.PowerOff(ctx)
		}); err != nil {
			log.Debugf("Failed to power off existing appliance for %s, try to remove anyway", err)
		}
	}
	// get the actual folder name before we delete it
	folder, err := vm.FolderName(d.ctx)
	if err != nil {
		log.Warnf("Failed to get actual folder name for VM. Will not attempt to delete additional data files in VM directory: %s", err)
	}

	_, err = tasks.WaitForResult(d.ctx, func(ctx context.Context) (tasks.ResultWaiter, error) {
		return vm.Destroy(ctx)
	})
	if err != nil {
		err = errors.Errorf("Failed to destroy vm %s: %s", vm.Reference(), err)
		return err
	}
	if _, err = d.deleteDatastoreFiles(d.session.Datastore, folder, true); err != nil {
		log.Warnf("VM path %s is not removed, %s", folder, err)
	}

	return nil
}

func (d *Dispatcher) deleteNetworkDevices(vmm *vm.VirtualMachine, conf *config.VirtualContainerHostConfigSpec) error {
	defer trace.End(trace.Begin(""))

	log.Infof("Removing appliance VM network devices")

	power, err := vmm.PowerState(d.ctx)
	if err != nil {
		log.Errorf("Failed to get vm power status %q: %s", vmm.Reference(), err)
		return err

	}
	if power != types.VirtualMachinePowerStatePoweredOff {
		if _, err = tasks.WaitForResult(d.ctx, func(ctx context.Context) (tasks.ResultWaiter, error) {
			return vmm.PowerOff(ctx)
		}); err != nil {
			log.Errorf("Failed to power off existing appliance for %s", err)
			return err
		}
	}

	devices, err := d.networkDevices(vmm)
	if err != nil {
		log.Errorf("Unable to get network devices: %s", err)
		return err
	}

	if len(devices) == 0 {
		log.Infof("No network device attached")
		return nil
	}
	// remove devices
	return vmm.RemoveDevice(d.ctx, false, devices...)
}

func TestDeleteExceptDisk(t *testing.T) {
	s := os.Getenv("DRONE")
	if s != "" {
		t.Skip("Skipping: test must be run in a VM")
	}

	ctx := context.Background()

	session := test.Session(ctx, t)
	defer session.Logout(ctx)

	host := test.PickRandomHost(ctx, session, t)

	uuid, err := sys.UUID()
	if err != nil {
		t.Fatalf("unable to get UUID for guest - used for VM name: %s", err)
	}
	name := fmt.Sprintf("%s-%d", uuid, rand.Intn(math.MaxInt32))

	moref, err := CreateVM(ctx, session, host, name)
	if err != nil {
		t.Fatalf("ERROR: %s", err)
	}
	// Wrap the result with our version of VirtualMachine
	vm := NewVirtualMachine(ctx, session, *moref)

	folder, err := vm.FolderName(ctx)
	if err != nil {
		t.Fatalf("ERROR: %s", err)
	}

	// generate the disk name
	diskName := fmt.Sprintf("%s/%s.vmdk", folder, folder)

	// Delete the VM but not it's disk
	_, err = tasks.WaitForResult(ctx, func(ctx context.Context) (tasks.ResultWaiter, error) {
		return vm.DeleteExceptDisks(ctx)
	})
	if err != nil {
		t.Fatalf("ERROR: %s", err)
	}

	// check that the disk still exists
	session.Datastore.Stat(ctx, diskName)
	if err != nil {
		t.Fatalf("Disk does not exist")
	}

	// clean up
	dm := object.NewVirtualDiskManager(session.Client.Client)

	task, err := dm.DeleteVirtualDisk(context.TODO(), diskName, nil)
	if err != nil {
		t.Fatalf("Unable to locate orphan vmdk: %s", err)
	}

	if err = task.Wait(context.TODO()); err != nil {
		t.Fatalf("Unable to remove orphan vmdk: %s", err)
	}
}

func (d *Dispatcher) destroyResourcePoolIfEmpty(conf *config.VirtualContainerHostConfigSpec) error {
	defer trace.End(trace.Begin(""))

	log.Infof("Removing Resource Pool %q", conf.Name)

	rpRef := conf.ComputeResources[len(conf.ComputeResources)-1]
	rp := compute.NewResourcePool(d.ctx, d.session, rpRef)

	var vms []*vm.VirtualMachine
	var err error
	if vms, err = rp.GetChildrenVMs(d.ctx, d.session); err != nil {
		err = errors.Errorf("Unable to get children vm of resource pool %q: %s", rp.Name(), err)
		return err
	}
	if len(vms) != 0 {
		err = errors.Errorf("Resource pool is not empty: %q", rp.Name())
		return err
	}
	if _, err := tasks.WaitForResult(d.ctx, func(ctx context.Context) (tasks.ResultWaiter, error) {
		return rp.Destroy(ctx)
	}); err != nil {
		return err
	}
	return nil
}

func createAppliance(ctx context.Context, sess *session.Session, conf *config.VirtualContainerHostConfigSpec, vConf *data.InstallerData, hasErr bool, t *testing.T) {
	var err error

	d := &Dispatcher{
		session: sess,
		ctx:     ctx,
		isVC:    sess.IsVC(),
		force:   false,
	}
	delete(conf.Networks, "bridge") // FIXME: cannot create bridge network in simulator
	if d.isVC {
		if d.vchVapp, err = d.createVApp(conf, vConf); err != nil {
			// FIXME: Got error: ServerFaultCode: ResourcePool:resourcepool-14 does not implement: CreateVApp. Simulator need to implement CreateVApp
			//			t.Errorf("Unable to create virtual app: %s", err)
		}
	}
	if d.vchPool, err = d.createResourcePool(conf, vConf); err != nil {
		t.Errorf("Unable to create resource pool: %s", err)
	}

	spec, err := d.createApplianceSpec(conf, vConf)
	if err != nil {
		t.Errorf("Unable to create appliance spec: %s", err)
		return
	}

	// create appliance VM
	info, err := tasks.WaitForResult(d.ctx, func(ctx context.Context) (tasks.ResultWaiter, error) {
		return d.session.Folders(ctx).VmFolder.CreateVM(ctx, *spec, d.vchPool, d.session.Host)
	})
	// get VM reference and save it
	moref := info.Result.(types.ManagedObjectReference)
	conf.SetMoref(&moref)
	obj, err := d.session.Finder.ObjectReference(d.ctx, moref)
	if err != nil {
		t.Errorf("Failed to reacquire reference to appliance VM after creation: %s", err)
		return
	}
	gvm, ok := obj.(*object.VirtualMachine)
	if !ok {
		t.Errorf("Required reference after appliance creation was not for a VM: %T", obj)
		return
	}

	vm2 := vm.NewVirtualMachineFromVM(d.ctx, d.session, gvm)
	uuid, err := vm2.UUID(d.ctx)
	if err != nil {
		t.Errorf("Failed to get VM UUID: %s", err)
		return
	}
	t.Logf("uuid: %s", uuid)

	// leverage create volume method to create image datastore
	conf.VolumeLocations["images-store"], _ = url.Parse(fmt.Sprintf("ds://LocalDS_0/VIC/%s/images", uuid))

	if err := d.createVolumeStores(conf); err != nil {
		t.Errorf("Unable to create volume stores: %s", err)
		return
	}
}

func CreateVM(ctx context.Context, session *session.Session, host *object.HostSystem, name string) (*types.ManagedObjectReference, error) {
	// Create the spec config
	specconfig := test.SpecConfig(session, name)

	// Create a linux guest
	linux, err := guest.NewLinuxGuest(ctx, session, specconfig)
	if err != nil {
		return nil, err
	}

	// Find the Virtual Machine folder that we use
	folders, err := session.Datacenter.Folders(ctx)
	if err != nil {
		return nil, err
	}
	parent := folders.VmFolder

	// Create the vm
	info, err := tasks.WaitForResult(ctx, func(ctx context.Context) (tasks.ResultWaiter, error) {
		return parent.CreateVM(ctx, *linux.Spec().Spec(), session.Pool, host)
	})
	if err != nil {
		return nil, err
	}

	moref := info.Result.(types.ManagedObjectReference)
	// Return the moRef
	return &moref, nil
}

func TestVM(t *testing.T) {

	s := os.Getenv("DRONE")
	if s != "" {
		t.Skip("Skipping: test must be run in a VM")
	}

	ctx := context.Background()

	session := test.Session(ctx, t)
	defer session.Logout(ctx)

	host := test.PickRandomHost(ctx, session, t)

	uuid, err := sys.UUID()
	if err != nil {
		t.Fatalf("unable to get UUID for guest - used for VM name: %s", err)
		return
	}
	name := fmt.Sprintf("%s-%d", uuid, rand.Intn(math.MaxInt32))

	moref, err := CreateVM(ctx, session, host, name)
	if err != nil {
		t.Fatalf("ERROR: %s", err)
	}
	// Wrap the result with our version of VirtualMachine
	vm := NewVirtualMachine(ctx, session, *moref)

	// Check the state
	state, err := vm.PowerState(ctx)
	if err != nil {
		t.Fatalf("ERROR: %s", err)
	}

	assert.Equal(t, types.VirtualMachinePowerStatePoweredOff, state)

	// Check VM name
	rname, err := vm.Name(ctx)
	if err != nil {
		t.Errorf("Failed to load VM name: %s", err)
	}
	assert.Equal(t, name, rname)

	// Get VM UUID
	ruuid, err := vm.UUID(ctx)
	if err != nil {
		t.Errorf("Failed to load VM UUID: %s", err)
	}
	t.Logf("Got UUID: %s", ruuid)

	// Destroy the vm
	_, err = tasks.WaitForResult(ctx, func(ctx context.Context) (tasks.ResultWaiter, error) {
		return vm.Destroy(ctx)
	})
	if err != nil {
		t.Fatalf("ERROR: %s", err)
	}
}

func (d *Dispatcher) deleteVMFSFiles(m *object.FileManager, ds *object.Datastore, dsPath string) error {
	defer trace.End(trace.Begin(dsPath))

	if _, err := tasks.WaitForResult(d.ctx, func(ctx context.Context) (tasks.Task, error) {
		return m.DeleteDatastoreFile(ctx, dsPath, d.session.Datacenter)
	}); err != nil {
		log.Debugf("Failed to delete %q: %s", dsPath, err)
	}
	return nil
}

// Start starts a container vm with the given params
func (c *Container) start(ctx context.Context) error {
	defer trace.End(trace.Begin("Container.start"))

	if c.vm == nil {
		return fmt.Errorf("vm not set")
	}
	// get existing state and set to starting
	// if there's a failure we'll revert to existing
	existingState := c.State
	c.State = StateStarting

	// Power on
	_, err := tasks.WaitForResult(ctx, func(ctx context.Context) (tasks.ResultWaiter, error) {
		return c.vm.PowerOn(ctx)
	})
	if err != nil {
		c.State = existingState
		return err
	}

	// guestinfo key that we want to wait for
	key := fmt.Sprintf("guestinfo..sessions|%s.started", c.ExecConfig.ID)
	var detail string

	// Wait some before giving up...
	ctx, cancel := context.WithTimeout(ctx, propertyCollectorTimeout)
	defer cancel()

	detail, err = c.vm.WaitForKeyInExtraConfig(ctx, key)
	if err != nil {
		c.State = existingState
		return fmt.Errorf("unable to wait for process launch status: %s", err.Error())
	}

	if detail != "true" {
		c.State = existingState
		return errors.New(detail)
	}

	return nil
}

func (c *Container) stop(ctx context.Context, waitTime *int32) error {
	defer trace.End(trace.Begin("Container.stop"))

	if c.vm == nil {
		return fmt.Errorf("vm not set")
	}
	// get existing state and set to stopping
	// if there's a failure we'll revert to existing
	existingState := c.State
	c.State = StateStopping

	err := c.shutdown(ctx, waitTime)
	if err == nil {
		return nil
	}

	log.Warnf("stopping %s via hard power off due to: %s", c.ExecConfig.ID, err)

	_, err = tasks.WaitForResult(ctx, func(ctx context.Context) (tasks.ResultWaiter, error) {
		return c.vm.PowerOff(ctx)
	})

	if err != nil {
		// It is possible the VM has finally shutdown in between, ignore the error in that case
		if terr, ok := err.(task.Error); ok {
			if serr, ok := terr.Fault().(*types.InvalidPowerState); ok {
				if serr.ExistingState == types.VirtualMachinePowerStatePoweredOff {
					log.Warnf("power off %s task skipped (state was already %s)", c.ExecConfig.ID, serr.ExistingState)
					return nil
				}
			}
		}
		c.State = existingState
	}
	return err
}

//.........这里部分代码省略.........

	// create a directory in the datastore
	// eat the error because we dont care if it exists
	fm.MakeDirectory(context.TODO(), imagestore, nil, true)

	vdm, err := NewDiskManager(context.TODO(), client)
	if err != nil && err.Error() == "can't find the hosting vm" {
		t.Skip("Skipping: test must be run in a VM")
	}

	if !assert.NoError(t, err) || !assert.NotNil(t, vdm) {
		return
	}

	diskSize := int64(1 << 10)
	parent, err := vdm.Create(context.TODO(), path.Join(imagestore, "scratch.vmdk"), diskSize)
	if !assert.NoError(t, err) {
		return
	}

	numChildren := 3
	children := make([]*VirtualDisk, numChildren)

	testString := "Ground control to Major Tom"
	writeSize := len(testString) / numChildren
	// Create children which inherit from eachother
	for i := 0; i < numChildren; i++ {

		p := path.Join(imagestore, fmt.Sprintf("child%d.vmdk", i))
		child, cerr := vdm.CreateAndAttach(context.TODO(), p, parent.DatastoreURI, 0, os.O_RDWR)
		if !assert.NoError(t, cerr) {
			return
		}

		children[i] = child

		// Write directly to the disk
		f, cerr := os.OpenFile(child.DevicePath, os.O_RDWR, os.FileMode(0777))
		if !assert.NoError(t, cerr) {
			return
		}

		start := i * writeSize
		end := start + writeSize

		if i == numChildren-1 {
			// last chunk, write to the end.
			_, cerr = f.WriteAt([]byte(testString[start:]), int64(start))
			if !assert.NoError(t, cerr) || !assert.NoError(t, f.Sync()) {
				return
			}

			// Try to read the whole string
			b := make([]byte, len(testString))
			f.Seek(0, 0)
			_, cerr = f.Read(b)
			if !assert.NoError(t, cerr) {
				return
			}

			//check against the test string
			if !assert.Equal(t, testString, string(b)) {
				return
			}

		} else {
			_, cerr = f.WriteAt([]byte(testString[start:end]), int64(start))
			if !assert.NoError(t, cerr) || !assert.NoError(t, f.Sync()) {
				return
			}
		}

		f.Close()

		cerr = vdm.Detach(context.TODO(), child)
		if !assert.NoError(t, cerr) {
			return
		}

		// use this image as the next parent
		parent = child
	}

	//	// Nuke the images
	//	for i := len(children) - 1; i >= 0; i-- {
	//		err = vdm.Delete(context.TODO(), children[i])
	//		if !assert.NoError(t, err) {
	//			return
	//		}
	//	}

	// Nuke the image store
	_, err = tasks.WaitForResult(context.TODO(), func(ctx context.Context) (tasks.ResultWaiter, error) {
		return fm.DeleteDatastoreFile(ctx, imagestore, nil)
	})

	if !assert.NoError(t, err) {
		return
	}
}

func (d *Dispatcher) createAppliance(conf *config.VirtualContainerHostConfigSpec, settings *data.InstallerData) error {
	defer trace.End(trace.Begin(""))

	log.Infof("Creating appliance on target")

	spec, err := d.createApplianceSpec(conf, settings)
	if err != nil {
		log.Errorf("Unable to create appliance spec: %s", err)
		return err
	}

	var info *types.TaskInfo
	// create appliance VM
	if d.isVC && d.vchVapp != nil {
		info, err = tasks.WaitForResult(d.ctx, func(ctx context.Context) (tasks.Task, error) {
			return d.vchVapp.CreateChildVM_Task(ctx, *spec, d.session.Host)
		})
	} else {
		// if vapp is not created, fall back to create VM under default resource pool
		folder := d.session.Folders(d.ctx).VmFolder
		info, err = tasks.WaitForResult(d.ctx, func(ctx context.Context) (tasks.Task, error) {
			return folder.CreateVM(ctx, *spec, d.vchPool, d.session.Host)
		})
	}

	if err != nil {
		log.Errorf("Unable to create appliance VM: %s", err)
		return err
	}
	if info.Error != nil || info.State != types.TaskInfoStateSuccess {
		log.Errorf("Create appliance reported: %s", info.Error.LocalizedMessage)
	}

	// get VM reference and save it
	moref := info.Result.(types.ManagedObjectReference)
	conf.SetMoref(&moref)
	obj, err := d.session.Finder.ObjectReference(d.ctx, moref)
	if err != nil {
		log.Errorf("Failed to reacquire reference to appliance VM after creation: %s", err)
		return err
	}
	gvm, ok := obj.(*object.VirtualMachine)
	if !ok {
		return fmt.Errorf("Required reference after appliance creation was not for a VM: %T", obj)
	}
	vm2 := vm.NewVirtualMachineFromVM(d.ctx, d.session, gvm)

	// update the displayname to the actual folder name used
	if d.vmPathName, err = vm2.FolderName(d.ctx); err != nil {
		log.Errorf("Failed to get canonical name for appliance: %s", err)
		return err
	}
	log.Debugf("vm folder name: %q", d.vmPathName)
	log.Debugf("vm inventory path: %q", vm2.InventoryPath)

	// create an extension to register the appliance as
	if err = d.GenerateExtensionName(conf, vm2); err != nil {
		return errors.Errorf("Could not generate extension name during appliance creation due to error: %s", err)
	}

	settings.Extension = types.Extension{
		Description: &types.Description{
			Label:   "VIC",
			Summary: "vSphere Integrated Containers Virtual Container Host",
		},
		Company: "VMware, Inc.",
		Version: "0.0",
		Key:     conf.ExtensionName,
	}

	conf.AddComponent("vicadmin", &executor.SessionConfig{
		User:  "vicadmin",
		Group: "vicadmin",
		Cmd: executor.Cmd{
			Path: "/sbin/vicadmin",
			Args: []string{
				"/sbin/vicadmin",
				"--dc=" + settings.DatacenterName,
				"--pool=" + settings.ResourcePoolPath,
				"--cluster=" + settings.ClusterPath,
			},
			Env: []string{
				"PATH=/sbin:/bin",
				"GOTRACEBACK=all",
			},
			Dir: "/home/vicadmin",
		},
		Restart: true,
	},
	)

	if conf.HostCertificate != nil {
		d.VICAdminProto = "https"
		d.DockerPort = fmt.Sprintf("%d", opts.DefaultTLSHTTPPort)
	} else {
		d.VICAdminProto = "http"
		d.DockerPort = fmt.Sprintf("%d", opts.DefaultHTTPPort)
	}

	personality := executor.Cmd{
//.........这里部分代码省略.........

func (d *Dispatcher) createAppliance(conf *metadata.VirtualContainerHostConfigSpec, settings *data.InstallerData) error {
	defer trace.End(trace.Begin(""))

	log.Infof("Creating appliance on target")

	spec, err := d.createApplianceSpec(conf, settings)
	if err != nil {
		log.Errorf("Unable to create appliance spec: %s", err)
		return err
	}

	// create test VM
	info, err := tasks.WaitForResult(d.ctx, func(ctx context.Context) (tasks.ResultWaiter, error) {
		return d.session.Folders(ctx).VmFolder.CreateVM(ctx, *spec, d.vchPool, d.session.Host)
	})

	if err != nil {
		log.Errorf("Unable to create appliance VM: %s", err)
		return err
	}
	if info.Error != nil || info.State != types.TaskInfoStateSuccess {
		log.Errorf("Create appliance reported: %s", info.Error.LocalizedMessage)
	}

	// get VM reference and save it
	moref := info.Result.(types.ManagedObjectReference)
	conf.SetMoref(&moref)
	obj, err := d.session.Finder.ObjectReference(d.ctx, moref)
	if err != nil {
		log.Errorf("Failed to reacquire reference to appliance VM after creation: %s", err)
		return err
	}
	gvm, ok := obj.(*object.VirtualMachine)
	if !ok {
		return fmt.Errorf("Required reference after appliance creation was not for a VM: %T", obj)
	}
	vm2 := vm.NewVirtualMachineFromVM(d.ctx, d.session, gvm)

	// update the displayname to the actual folder name used
	if d.vmPathName, err = vm2.FolderName(d.ctx); err != nil {
		log.Errorf("Failed to get canonical name for appliance: %s", err)
		return err
	}
	log.Debugf("vm folder name: %s", d.vmPathName)
	log.Debugf("vm inventory path: %s", vm2.InventoryPath)

	// create an extension to register the appliance as
	if err = d.GenerateExtensionName(conf); err != nil {
		return errors.Errorf("Could not generate extension name during appliance creation due to error: %s", err)
	}

	settings.Extension = types.Extension{
		Description: &types.Description{
			Label:   "VIC",
			Summary: "vSphere Integrated Containers Virtual Container Host",
		},
		Company: "VMware, Inc.",
		Version: "0.0",
		Key:     conf.ExtensionName,
	}

	conf.AddComponent("vicadmin", &metadata.SessionConfig{
		Cmd: metadata.Cmd{
			Path: "/sbin/vicadmin",
			Args: []string{
				"/sbin/vicadmin",
				"-docker-host=unix:///var/run/docker.sock",
				// FIXME: hack during config migration
				"-insecure",
				"-sdk=" + conf.Target.String(),
				"-ds=" + conf.ImageStores[0].Host,
				"-cluster=" + settings.ClusterPath,
				"-pool=" + settings.ResourcePoolPath,
				"-vm-path=" + vm2.InventoryPath,
			},
			Env: []string{
				"PATH=/sbin:/bin",
			},
		},
	},
	)

	if conf.HostCertificate != nil {
		d.VICAdminProto = "https"
		d.DockerPort = "2376"
	} else {
		d.VICAdminProto = "http"
		d.DockerPort = "2375"
	}

	conf.AddComponent("docker-personality", &metadata.SessionConfig{
		Cmd: metadata.Cmd{
			Path: "/sbin/docker-engine-server",
			Args: []string{
				"/sbin/docker-engine-server",
				//FIXME: hack during config migration
				"-serveraddr=0.0.0.0",
				"-port=" + d.DockerPort,
				"-port-layer-port=8080",
			},
//.........这里部分代码省略.........

func TestVMAttributes(t *testing.T) {

	ctx := context.Background()

	session := test.Session(ctx, t)
	defer session.Logout(ctx)

	host := test.PickRandomHost(ctx, session, t)

	uuid, err := sys.UUID()
	if err != nil {
		t.Fatalf("unable to get UUID for guest - used for VM name: %s", err)
		return
	}
	ID := fmt.Sprintf("%s-%d", uuid, rand.Intn(math.MaxInt32))

	moref, err := CreateVM(ctx, session, host, ID)
	if err != nil {
		t.Fatalf("ERROR: %s", err)
	}
	// Wrap the result with our version of VirtualMachine
	vm := NewVirtualMachine(ctx, session, *moref)

	folder, err := vm.FolderName(ctx)
	if err != nil {
		t.Fatalf("ERROR: %s", err)
	}

	name, err := vm.Name(ctx)
	if err != nil {
		t.Fatalf("ERROR: %s", err)
	}
	assert.Equal(t, name, folder)

	_, err = tasks.WaitForResult(ctx, func(ctx context.Context) (tasks.ResultWaiter, error) {
		return vm.PowerOn(ctx)
	})
	if err != nil {
		t.Fatalf("ERROR: %s", err)
	}

	if guest, err := vm.FetchExtraConfig(ctx); err != nil {
		t.Fatalf("ERROR: %s", err)
	} else {
		assert.NotEmpty(t, guest)
	}
	defer func() {
		// Destroy the vm
		_, err = tasks.WaitForResult(ctx, func(ctx context.Context) (tasks.ResultWaiter, error) {
			return vm.PowerOff(ctx)
		})
		if err != nil {
			t.Fatalf("ERROR: %s", err)
		}
		_, err = tasks.WaitForResult(ctx, func(ctx context.Context) (tasks.ResultWaiter, error) {
			return vm.Destroy(ctx)
		})
		if err != nil {
			t.Fatalf("ERROR: %s", err)
		}
	}()
}

func (c *Container) Remove(ctx context.Context, sess *session.Session) error {
	defer trace.End(trace.Begin("Container.Remove"))
	c.Lock()
	defer c.Unlock()

	if c.vm == nil {
		return NotFoundError{}
	}

	// check state first
	if c.State == StateRunning {
		return RemovePowerError{fmt.Errorf("Container is powered on")}
	}

	// get existing state and set to removing
	// if there's a failure we'll revert to existing
	existingState := c.State
	c.State = StateRemoving

	// get the folder the VM is in
	url, err := c.vm.DSPath(ctx)
	if err != nil {

		// handle the out-of-band removal case
		if soap.IsSoapFault(err) {
			fault := soap.ToSoapFault(err).VimFault()
			if _, ok := fault.(types.ManagedObjectNotFound); ok {
				containers.Remove(c.ExecConfig.ID)
				return NotFoundError{}
			}
		}

		log.Errorf("Failed to get datastore path for %s: %s", c.ExecConfig.ID, err)
		c.State = existingState
		return err
	}
	// FIXME: was expecting to find a utility function to convert to/from datastore/url given
	// how widely it's used but couldn't - will ask around.
	dsPath := fmt.Sprintf("[%s] %s", url.Host, url.Path)

	//removes the vm from vsphere, but detaches the disks first
	_, err = tasks.WaitForResult(ctx, func(ctx context.Context) (tasks.ResultWaiter, error) {
		return c.vm.DeleteExceptDisks(ctx)
	})
	if err != nil {
		c.State = existingState
		return err
	}

	// remove from datastore
	fm := object.NewFileManager(c.vm.Client.Client)

	if _, err = tasks.WaitForResult(ctx, func(ctx context.Context) (tasks.ResultWaiter, error) {
		return fm.DeleteDatastoreFile(ctx, dsPath, sess.Datacenter)
	}); err != nil {
		c.State = existingState
		log.Debugf("Failed to delete %s, %s", dsPath, err)
	}

	//remove container from cache
	containers.Remove(c.ExecConfig.ID)
	return nil
}

func (d *Dispatcher) CreateVCH(conf *config.VirtualContainerHostConfigSpec, settings *data.InstallerData) error {
	defer trace.End(trace.Begin(conf.Name))

	var err error

	if err = d.checkExistence(conf, settings); err != nil {
		return err
	}

	if d.isVC && !settings.UseRP {
		if d.vchVapp, err = d.createVApp(conf, settings); err != nil {
			detail := fmt.Sprintf("Creating virtual app failed: %s", err)
			if !d.force {
				return errors.New(detail)
			}

			log.Error(detail)
			log.Errorf("Deploying vch under parent pool %q, (--force=true)", settings.ResourcePoolPath)
			d.vchPool = d.session.Pool
			conf.ComputeResources = append(conf.ComputeResources, d.vchPool.Reference())
		}
	} else {
		if d.vchPool, err = d.createResourcePool(conf, settings); err != nil {
			detail := fmt.Sprintf("Creating resource pool failed: %s", err)
			if !d.force {
				return errors.New(detail)
			}

			log.Error(detail)
			log.Errorf("Deploying vch under parent pool %q, (--force=true)", settings.ResourcePoolPath)
			d.vchPool = d.session.Pool
			conf.ComputeResources = append(conf.ComputeResources, d.vchPool.Reference())
		}
	}

	if err = d.createBridgeNetwork(conf); err != nil {
		return err
	}

	if err = d.createVolumeStores(conf); err != nil {
		return errors.Errorf("Exiting because we could not create volume stores due to error: %s", err)
	}

	if err = d.createAppliance(conf, settings); err != nil {
		return errors.Errorf("Creating the appliance failed with %s. Exiting...", err)
	}

	if err = d.uploadImages(settings.ImageFiles); err != nil {
		return errors.Errorf("Uploading images failed with %s. Exiting...", err)
	}

	if d.session.IsVC() {
		if err = d.RegisterExtension(conf, settings.Extension); err != nil {
			return errors.Errorf("Error registering VCH vSphere extension: %s", err)
		}
	}
	_, err = tasks.WaitForResult(d.ctx, func(ctx context.Context) (tasks.ResultWaiter, error) {
		return d.appliance.PowerOn(ctx)
	})

	if err != nil {
		return errors.Errorf("Failed to power on appliance %s. Exiting...", err)
	}

	if err = d.ensureApplianceInitializes(conf); err != nil {
		return errors.Errorf("%s. Exiting...", err)
	}

	// wait till the appliance components are fully initialized
	if err = d.ensureComponentsInitialize(conf); err != nil {
		return errors.Errorf("%s. Exiting...", err)
	}

	return nil
}

func TestVolumeCreateListAndRestart(t *testing.T) {
	client := datastore.Session(context.TODO(), t)
	if client == nil {
		return
	}

	ctx := context.TODO()

	// Create the backing store on vsphere
	vsVolumeStore, err := NewVolumeStore(ctx, client)
	if !assert.NoError(t, err) || !assert.NotNil(t, vsVolumeStore) {
		return
	}

	// Root our datastore
	testStorePath := datastore.TestName("voltest")
	ds, err := datastore.NewHelper(ctx, client, client.Datastore, testStorePath)
	if !assert.NoError(t, err) || !assert.NotNil(t, ds) {
		return
	}

	// Add a volume store and give it a name ("testStoreName")
	volumeStore, err := vsVolumeStore.AddStore(ctx, ds, "testStoreName")
	if !assert.NoError(t, err) || !assert.NotNil(t, volumeStore) {
		return
	}

	// test we can list it
	m, err := vsVolumeStore.VolumeStoresList(ctx)
	if !assert.NoError(t, err) || !assert.NotNil(t, m) {
		return
	}

	// test the returned url matches
	s, ok := m["testStoreName"]
	if !assert.True(t, ok) || !assert.Equal(t, testStorePath, filepath.Base(s.String())) {
		return
	}

	// Clean up the mess
	defer func() {
		fm := object.NewFileManager(client.Vim25())
		tasks.WaitForResult(context.TODO(), func(ctx context.Context) (tasks.ResultWaiter, error) {
			return fm.DeleteDatastoreFile(ctx, client.Datastore.Path(testStorePath), client.Datacenter)
		})
	}()

	// Create the cache
	cache, err := portlayer.NewVolumeLookupCache(ctx, vsVolumeStore)
	if !assert.NoError(t, err) || !assert.NotNil(t, cache) {
		return
	}

	// Create the volumes (in parallel)
	numVols := 5
	wg := &sync.WaitGroup{}
	wg.Add(numVols)
	volumes := make(map[string]*portlayer.Volume)
	for i := 0; i < numVols; i++ {
		go func(idx int) {
			defer wg.Done()
			ID := fmt.Sprintf("testvolume-%d", idx)

			// add some metadata if i is even
			var info map[string][]byte

			if idx%2 == 0 {
				info = make(map[string][]byte)
				info[ID] = []byte(ID)
			}

			outVol, err := cache.VolumeCreate(ctx, ID, volumeStore, 10240, info)
			if !assert.NoError(t, err) || !assert.NotNil(t, outVol) {
				return
			}

			volumes[ID] = outVol
		}(i)
	}

	wg.Wait()

	// list using the datastore (skipping the cache)
	outVols, err := vsVolumeStore.VolumesList(ctx)
	if !assert.NoError(t, err) || !assert.NotNil(t, outVols) || !assert.Equal(t, numVols, len(outVols)) {
		return
	}

	for _, outVol := range outVols {
		if !assert.Equal(t, volumes[outVol.ID], outVol) {
			return
		}
	}

	// Test restart

	// Create a new vs and cache to the same datastore (simulating restart) and compare
	secondVStore, err := NewVolumeStore(ctx, client)
	if !assert.NoError(t, err) || !assert.NotNil(t, vsVolumeStore) {
		return
//.........这里部分代码省略.........

// Remove removes a containerVM after detaching the disks
func (c *Container) Remove(ctx context.Context, sess *session.Session) error {
	defer trace.End(trace.Begin(c.ExecConfig.ID))
	c.m.Lock()
	defer c.m.Unlock()

	if c.vm == nil {
		return NotFoundError{}
	}

	// check state first
	if c.state == StateRunning {
		return RemovePowerError{fmt.Errorf("Container is powered on")}
	}

	// get existing state and set to removing
	// if there's a failure we'll revert to existing
	existingState := c.updateState(StateRemoving)

	// get the folder the VM is in
	url, err := c.vm.DSPath(ctx)
	if err != nil {

		// handle the out-of-band removal case
		if soap.IsSoapFault(err) {
			fault := soap.ToSoapFault(err).VimFault()
			if _, ok := fault.(types.ManagedObjectNotFound); ok {
				Containers.Remove(c.ExecConfig.ID)
				return NotFoundError{}
			}
		}

		log.Errorf("Failed to get datastore path for %s: %s", c.ExecConfig.ID, err)
		c.updateState(existingState)
		return err
	}
	// FIXME: was expecting to find a utility function to convert to/from datastore/url given
	// how widely it's used but couldn't - will ask around.
	dsPath := fmt.Sprintf("[%s] %s", url.Host, url.Path)

	//removes the vm from vsphere, but detaches the disks first
	_, err = c.vm.WaitForResult(ctx, func(ctx context.Context) (tasks.Task, error) {
		return c.vm.DeleteExceptDisks(ctx)
	})
	if err != nil {
		f, ok := err.(types.HasFault)
		if !ok {
			c.updateState(existingState)
			return err
		}
		switch f.Fault().(type) {
		case *types.InvalidState:
			log.Warnf("container VM is in invalid state, unregistering")
			if err := c.vm.Unregister(ctx); err != nil {
				log.Errorf("Error while attempting to unregister container VM: %s", err)
				return err
			}
		default:
			log.Debugf("Fault while attempting to destroy vm: %#v", f.Fault())
			c.updateState(existingState)
			return err
		}
	}

	// remove from datastore
	fm := object.NewFileManager(c.vm.Client.Client)

	if _, err = tasks.WaitForResult(ctx, func(ctx context.Context) (tasks.Task, error) {
		return fm.DeleteDatastoreFile(ctx, dsPath, sess.Datacenter)
	}); err != nil {
		// at this phase error doesn't matter. Just log it.
		log.Debugf("Failed to delete %s, %s", dsPath, err)
	}

	//remove container from cache
	Containers.Remove(c.ExecConfig.ID)
	return nil
}