func (d *driver) createVolumeHandleSnapshotID(
	size *int64, snapshotID string, fields map[string]interface{}) error {
	if snapshotID == "" {
		return nil
	}
	snapshots, err := d.GetSnapshot("", snapshotID, "")
	if err != nil {
		return goof.WithFieldsE(fields, "error getting snapshot", err)
	}

	if len(snapshots) == 0 {
		return goof.WithFields(fields, "snapshot array is empty")
	}

	volSize := snapshots[0].VolumeSize
	sizeInt, err := strconv.Atoi(volSize)
	if err != nil {
		f := goof.Fields{
			"volumeSize": volSize,
		}
		for k, v := range fields {
			f[k] = v
		}
		return goof.WithFieldsE(f, "error casting volume size", err)
	}
	*size = int64(sizeInt)
	return nil
}

func (d *driver) getVolume(volumeID, volumeName string) ([]xtio.Volume, error) {
	fields := eff(map[string]interface{}{
		"volumeID":   volumeID,
		"volumeName": volumeName,
	})

	var volumes []xtio.Volume
	if volumeID != "" || volumeName != "" {
		volume, err := d.client.GetVolume(volumeID, volumeName)
		if err != nil {
			return nil, goof.WithFieldsE(fields, "error getting volume", err)
		}
		volumes = append(volumes, volume)
	} else {
		allVolumes, err := d.client.GetVolumes()
		if err != nil {
			return nil, goof.WithFieldsE(fields, "error getting volumes", err)
		}
		for _, volume := range allVolumes {
			hrefFields := strings.Split(volume.Href, "/")
			index, _ := strconv.Atoi(hrefFields[len(hrefFields)-1])
			volumes = append(volumes,
				xtio.VolumeCtorNameIndex(volume.Name, index))
		}
	}
	return volumes, nil
}

func (d *driver) VolumeRemove(
	ctx types.Context,
	volumeID string,
	opts types.Store) error {

	fields := eff(map[string]interface{}{
		"volumeId": volumeID,
	})

	var err error
	var volumes []*siotypes.Volume

	if volumes, err = d.getVolume(volumeID, "", 0); err != nil {
		return goof.WithFieldsE(fields, "error getting volume", err)
	}

	targetVolume := sio.NewVolume(d.client)
	targetVolume.Volume = volumes[0]

	if err = targetVolume.RemoveVolume("ONLY_ME"); err != nil {
		return goof.WithFieldsE(fields, "error removing volume", err)
	}

	log.WithFields(fields).Debug("removed volume")
	return nil
}

// 	// VolumeRemove removes a volume.
func (d *driver) VolumeRemove(
	ctx types.Context,
	volumeID string,
	opts types.Store) error {
	fields := eff(map[string]interface{}{
		"volumeId": volumeID,
	})
	if volumeID == "" {
		return goof.WithFields(fields, "volumeId is required")
	}

	attached, err := d.volumeAttached(ctx, volumeID)
	if err != nil {
		return goof.WithFieldsE(fields, "error retrieving attachment status", err)
	}

	if attached {
		_, err := d.VolumeDetach(ctx, volumeID, &types.VolumeDetachOpts{})
		if err != nil {
			return goof.WithFieldsE(fields, "error detaching before volume removal", err)
		}
	}

	res := volumes.Delete(d.clientBlockStorage, volumeID)
	if res.Err != nil {
		return goof.WithFieldsE(fields, "error removing volume", res.Err)
	}

	return nil
}

func (d *driver) RemoveVolume(volumeID string) error {

	fields := eff(map[string]interface{}{
		"volumeId": volumeID,
	})

	if volumeID == "" {
		return goof.WithFields(fields, "volumeId is required")
	}

	var err error
	var volumes []*types.Volume

	if volumes, err = d.getVolume(volumeID, "", false); err != nil {
		return goof.WithFieldsE(fields, "error getting volume", err)
	}

	targetVolume := goscaleio.NewVolume(d.client)
	targetVolume.Volume = volumes[0]

	if err = targetVolume.RemoveVolume("ONLY_ME"); err != nil {
		return goof.WithFieldsE(fields, "error removing volume", err)
	}

	log.WithFields(fields).Debug("removed volume")
	return nil
}

func (d *driver) Init(context types.Context, config gofig.Config) error {
	d.config = config

	fields := eff(map[string]interface{}{})
	var err error

	if d.instanceID, err = d.getInstanceID(); err != nil {
		return err
	}

	fields["moduleName"] = context
	fields["instanceId"] = d.instanceID

	if d.region, err = d.getInstanceRegion(); err != nil {
		return err
	}

	fields["region"] = d.region
	d.region = strings.ToUpper(d.region)

	authOpts := d.getAuthOptions()

	fields["identityEndpoint"] = d.authURL()
	fields["userId"] = d.userID()
	fields["userName"] = d.userName()
	if d.password() == "" {
		fields["password"] = ""
	} else {
		fields["password"] = "******"
	}
	fields["tenantId"] = d.tenantID()
	fields["tenantName"] = d.tenantName()
	fields["domainId"] = d.domainID()
	fields["domainName"] = d.domainName()

	if d.provider, err = openstack.AuthenticatedClient(authOpts); err != nil {
		return goof.WithFieldsE(fields,
			"error getting authenticated client", err)
	}

	if d.client, err = openstack.NewComputeV2(d.provider,
		gophercloud.EndpointOpts{Region: d.region}); err != nil {
		goof.WithFieldsE(fields, "error getting newComputeV2", err)
	}

	if d.clientBlockStorage, err = openstack.NewBlockStorageV1(d.provider,
		gophercloud.EndpointOpts{Region: d.region}); err != nil {
		return goof.WithFieldsE(fields,
			"error getting newBlockStorageV1", err)
	}

	log.WithFields(fields).Info("storage driver initialized")
	return nil

}

func (d *driver) CreateVolume(
	notUsed bool,
	volumeName, volumeID, snapshotID, NUvolumeType string,
	NUIOPS, size int64, NUavailabilityZone string) (*core.Volume, error) {

	fields := eff(map[string]interface{}{
		"volumeName": volumeName,
		"volumeId":   volumeID,
	})

	log.WithFields(fields).Debug("Creating volume")

	if volumeID == "" && volumeName == "" {
		return nil, goof.New("Cannot create a volume.  No volume name or ID provided")
	}
	if volumeName == "" {
		volumeName = volumeID
	}
	_, err := d.client.CreateVolume(volumeName)
	if err != nil {
		return nil, goof.WithFieldsE(
			eff(goof.Fields{"volumeName": volumeName}), "Error creating volume", err)
	}

	// Set or update the quota for volume
	if d.quotas() {
		quota, err := d.client.GetQuota(volumeName)
		if quota == nil {
			err = d.client.SetQuotaSize(volumeName, size*bytesPerGb)
			if err != nil {
				// TODO: not sure how to handle this situation.  Delete created volume
				// and return an error?  Ignore and continue?
				return nil, goof.WithFieldsE(
					eff(goof.Fields{"volumeName": volumeName}), "Error setting quota for new volume", err)
			}
		} else {
			err = d.client.UpdateQuotaSize(volumeName, size*bytesPerGb)
			if err != nil {
				// TODO: not sure how to handle this situation.  Delete created volume
				// and return an error?  Ignore and continue?
				return nil, goof.WithFieldsE(
					eff(goof.Fields{"volumeName": volumeName}), "Error updating quota for existing volume", err)
			}
		}
	}

	volumes, _ := d.GetVolume(volumeID, volumeName)
	if volumes == nil || err != nil {
		return nil, goof.WithFieldsE(
			eff(goof.Fields{"volumeName": volumeName}), "Error getting volume", err)
	}

	return volumes[0], nil
}

func (d *driver) getVolume(
	volumeID, volumeName string) (volumesRet []volumes.Volume, err error) {

	if volumeID != "" {
		volume, err := volumes.Get(d.clientBlockStorage, volumeID).Extract()
		if err != nil {
			return []volumes.Volume{},
				goof.WithFieldsE(eff(goof.Fields{
					"volumeId":   volumeID,
					"volumeName": volumeName}),
					"error getting volumes", err)
		}
		volumesRet = append(volumesRet, *volume)
	} else {
		listOpts := &volumes.ListOpts{
		//Name:       volumeName,
		}

		allPages, err := volumes.List(d.clientBlockStorage, listOpts).AllPages()
		if err != nil {
			return []volumes.Volume{},
				goof.WithFieldsE(eff(goof.Fields{
					"volumeId":   volumeID,
					"volumeName": volumeName}),
					"error listing volumes", err)
		}
		volumesRet, err = volumes.ExtractVolumes(allPages)
		if err != nil {
			return []volumes.Volume{},
				goof.WithFieldsE(eff(goof.Fields{
					"volumeId":   volumeID,
					"volumeName": volumeName}),
					"error extracting volumes", err)
		}

		var volumesRetFiltered []volumes.Volume
		if volumeName != "" {
			var found bool
			for _, volume := range volumesRet {
				if volume.Name == volumeName {
					volumesRetFiltered = append(volumesRetFiltered, volume)
					found = true
					break
				}
			}
			if !found {
				return []volumes.Volume{}, nil
			}
			volumesRet = volumesRetFiltered
		}
	}

	return volumesRet, nil
}

func (d *driver) CreateSnapshot(
	notUsed bool,
	snapshotName, volumeID, description string) ([]*core.Snapshot, error) {

	fields := eff(map[string]interface{}{
		"volumeID":     volumeID,
		"snapshotName": snapshotName,
		"description":  description,
	})

	if snapshotName == "" {
		return nil, goof.New("no snapshot name specified")
	}

	volumes, err := d.GetVolume("", snapshotName)
	if err != nil {
		return nil, err
	}

	if len(volumes) > 0 {
		return nil, goof.WithFieldsE(fields, "volume name already exists", err)
	}

	snapshotDef := &types.SnapshotDef{
		VolumeID:     volumeID,
		SnapshotName: snapshotName,
	}

	var snapshotDefs []*types.SnapshotDef
	snapshotDefs = append(snapshotDefs, snapshotDef)
	snapshotVolumesParam := &types.SnapshotVolumesParam{
		SnapshotDefs: snapshotDefs,
	}

	var snapshot []*core.Snapshot
	var snapshotVolumes *types.SnapshotVolumesResp

	if snapshotVolumes, err =
		d.system.CreateSnapshotConsistencyGroup(
			snapshotVolumesParam); err != nil {
		return nil, goof.WithFieldsE(fields, "failed to create snapshot", err)
	}

	if snapshot, err = d.GetSnapshot(
		"", snapshotVolumes.VolumeIDList[0], ""); err != nil {
		return nil, goof.WithFieldsE(fields, "error getting new snapshot", err)
	}

	log.WithFields(log.Fields{
		"provider": providerName,
		"snapshot": snapshot}).Debug("created snapshot")
	return snapshot, nil

}

func (d *driver) Init(r *core.RexRay) error {

	d.r = r

	fields := eff(map[string]interface{}{
		"userName": d.userName(),
		"smisHost": d.smisHost(),
		"smisPort": d.smisPort(),
		"insecure": d.insecure(),
		"sid":      d.sid(),
	})

	if d.password() == "" {
		fields["password"] = ""
	} else {
		fields["password"] = "******"
	}

	var err error
	if d.client, err = govmax.New(
		d.smisHost(),
		d.smisPort(),
		d.insecure(),
		d.userName(),
		d.password()); err != nil {
		return goof.WithFieldsE(fields,
			"error creating govmax client", err)
	}

	d.arrayID = d.sid()
	d.volPrefix = d.volumePrefix()

	vmFields := eff(map[string]interface{}{
		"userName": d.vmhUserName(),
		"smisHost": d.vmhHost(),
		"insecure": d.vmhInsecure(),
	})

	if d.vmh, err = govmax.NewVMHost(
		d.vmhInsecure(),
		d.vmhHost(),
		d.vmhUserName(),
		d.vmhPassword(),
	); err != nil {
		return goof.WithFieldsE(vmFields,
			"error retrieving VM host info", err)
	}

	d.instanceID = d.vmh.Vm.Reference().Value

	log.WithField("provider", providerName).Info("storage driver initialized")

	return nil
}

func (d *driver) getVolume(
	ctx types.Context,
	volumeID, volumeName string,
	attachments types.VolumeAttachmentsTypes) ([]*types.Volume, error) {

	var volumesRet []volumes.Volume
	fields := eff(goof.Fields{
		"moduleName": ctx,
		"volumeId":   volumeID,
		"volumeName": volumeName})

	if volumeID != "" {
		volume, err := volumes.Get(d.clientBlockStorage, volumeID).Extract()
		if err != nil {
			return nil,
				goof.WithFieldsE(fields, "error getting volumes", err)
		}
		volumesRet = append(volumesRet, *volume)
	} else {
		listOpts := &volumes.ListOpts{
		//Name:       volumeName,
		}

		allPages, err := volumes.List(d.clientBlockStorage, listOpts).AllPages()
		if err != nil {
			return nil,
				goof.WithFieldsE(fields, "error listing volumes", err)
		}
		volumesRet, err = volumes.ExtractVolumes(allPages)
		if err != nil {
			return nil,
				goof.WithFieldsE(fields, "error extracting volumes", err)
		}

		var volumesRetFiltered []volumes.Volume
		if volumeName != "" {
			for _, volumer := range volumesRet { //volumer avoids any namespace confict
				if volumer.Name == volumeName {
					volumesRetFiltered = append(volumesRetFiltered, volumer)
					break
				}
			}
			volumesRet = volumesRetFiltered
		}
	}
	//now cast from []volumes.Volume to []types.Volume
	var volumesSD []*types.Volume
	for _, volume := range volumesRet {
		volumesSD = append(volumesSD, translateVolume(&volume, attachments))
	}
	return volumesSD, nil
}

func (d *driver) Init(r *core.RexRay) error {

	d.r = r
	d.volumesByNaa = map[string]xtio.Volume{}

	fields := eff(map[string]interface{}{
		"endpoint":         d.endpoint(),
		"userName":         d.userName(),
		"deviceMapper":     d.deviceMapper(),
		"multipath":        d.multipath(),
		"remoteManagement": d.remoteManagement(),
		"insecure":         d.insecure(),
	})

	if d.password() == "" {
		fields["password"] = ""
	} else {
		fields["password"] = "******"
	}

	if !isXtremIOAttached() && !d.remoteManagement() {
		return goof.WithFields(fields, "device not detected")
	}

	var err error

	if d.client, err = xtio.NewClientWithArgs(
		d.endpoint(),
		d.insecure(),
		d.userName(),
		d.password()); err != nil {
		return goof.WithFieldsE(fields,
			"error creating xtremio client", err)
	}

	if !d.remoteManagement() {
		var iqn string
		if iqn, err = getIQN(); err != nil {
			return goof.WithFieldsE(fields,
				"error getting IQN", err)
		}
		if d.initiator, err = d.client.GetInitiator("", iqn); err != nil {
			return goof.WithFieldsE(fields,
				"error getting initiator", err)
		}
	}

	log.WithField("provider", providerName).Info("storage driver initialized")

	return nil
}

func (d *driver) createVolumeHandleVolumeID(
	availabilityZone, snapshotID, volumeID *string,
	size *int64,
	fields map[string]interface{}) ([]*core.Volume, error) {

	if *volumeID == "" {
		return nil, nil
	}

	var err error
	var volume []*core.Volume

	if volume, err = d.GetVolume(*volumeID, ""); err != nil {
		return nil, goof.WithFieldsE(fields, "error getting volumes", err)
	}

	if len(volume) == 0 {
		return nil, goof.WithFieldsE(fields, "", errors.ErrNoVolumesReturned)
	}

	volSize := volume[0].Size
	sizeInt, err := strconv.Atoi(volSize)
	if err != nil {
		f := goof.Fields{
			"volumeSize": volSize,
		}
		for k, v := range fields {
			f[k] = v
		}
		return nil,
			goof.WithFieldsE(f, "error casting volume size", err)
	}
	*size = int64(sizeInt)

	*volumeID = volume[0].VolumeID
	snapshot, err := d.CreateSnapshot(
		false, fmt.Sprintf("temp-%s", *volumeID), *volumeID, "")
	if err != nil {
		return nil,
			goof.WithFields(fields, "error creating snapshot")
	}

	*snapshotID = snapshot[0].SnapshotID

	if *availabilityZone == "" {
		*availabilityZone = volume[0].AvailabilityZone
	}

	return volume, nil
}

func (d *driver) DetachVolume(
	runAsync bool, volumeID, instanceID string, force bool) error {

	fields := eff(map[string]interface{}{
		"runAsync":   runAsync,
		"volumeId":   volumeID,
		"instanceId": instanceID,
	})

	if volumeID == "" {
		return goof.WithFields(fields, "volumeId is required")
	}
	volume, err := d.GetVolume(volumeID, "")
	if err != nil {
		return goof.WithFieldsE(fields, "error getting volume", err)
	}

	if len(volume) == 0 {
		return goof.WithFields(fields, "no volumes returned")
	}

	if len(volume[0].Attachments) == 0 {
		return nil
	}

	fields["instanceId"] = volume[0].Attachments[0].InstanceID
	if force {
		if resp := volumeactions.ForceDetach(d.clientBlockStoragev2, volumeID); resp.Err != nil {
			log.Info(fmt.Sprintf("%+v", resp.Err))
			return goof.WithFieldsE(fields, "error forcing detach volume", resp.Err)
		}
	} else {
		if resp := volumeattach.Delete(
			d.client, volume[0].Attachments[0].InstanceID, volumeID); resp.Err != nil {
			return goof.WithFieldsE(fields, "error detaching volume", resp.Err)
		}
	}

	if !runAsync {
		log.WithFields(fields).Debug("waiting for volume to detach")
		err = d.waitVolumeDetach(volumeID)
		if err != nil {
			return goof.WithFieldsE(
				fields, "error waiting for volume to detach", err)
		}
	}

	log.WithFields(fields).Debug("volume detached")
	return nil
}

func (d *driver) AttachVolume(
	runAsync bool, volumeID, instanceID string, force bool) ([]*core.VolumeAttachment, error) {

	fields := eff(map[string]interface{}{
		"runAsync":   runAsync,
		"volumeId":   volumeID,
		"instanceId": instanceID,
	})

	nextDeviceName, err := d.GetDeviceNextAvailable()
	if err != nil {
		return nil, goof.WithFieldsE(
			fields, "error getting next available device", err)
	}

	if force {
		if err := d.DetachVolume(false, volumeID, "", true); err != nil {
			return nil, err
		}
	}

	options := &volumeattach.CreateOpts{
		Device:   nextDeviceName,
		VolumeID: volumeID,
	}

	_, err = volumeattach.Create(d.client, instanceID, options).Extract()
	if err != nil {
		return nil, goof.WithFieldsE(
			fields, "error attaching volume", err)
	}

	if !runAsync {
		log.WithFields(fields).Debug("waiting for volume to attach")
		err = d.waitVolumeAttach(volumeID)
		if err != nil {
			return nil, goof.WithFieldsE(
				fields, "error waiting for volume to detach", err)
		}
	}

	volumeAttachment, err := d.GetVolumeAttach(volumeID, instanceID)
	if err != nil {
		return nil, err
	}

	log.WithFields(fields).Debug("volume attached")
	return volumeAttachment, nil
}

func (d *driver) CreateSnapshot(
	runAsync bool,
	snapshotName, volumeID, description string) ([]*core.Snapshot, error) {

	fields := eff(map[string]interface{}{
		"runAsync":     runAsync,
		"snapshotName": snapshotName,
		"volumeId":     volumeID,
		"description":  description,
	})

	opts := snapshots.CreateOpts{
		Name:        snapshotName,
		VolumeID:    volumeID,
		Description: description,
		Force:       true,
	}

	resp, err := snapshots.Create(d.clientBlockStorage, opts).Extract()
	if err != nil {
		return nil,
			goof.WithFieldsE(fields, "error creating snapshot", err)
	}

	if !runAsync {
		log.Debug("waiting for snapshot creation to complete")
		err = snapshots.WaitForStatus(d.clientBlockStorage, resp.ID, "available", 120)
		if err != nil {
			return nil,
				goof.WithFieldsE(fields,
					"error waiting for snapshot creation to complete", err)
		}
	}

	snapshot, err := d.GetSnapshot("", resp.ID, "")
	if err != nil {
		return nil, err
	}

	log.WithFields(log.Fields{
		"runAsync":     runAsync,
		"snapshotName": snapshotName,
		"volumeId":     volumeID,
		"description":  description}).Debug("created snapshot")

	return snapshot, nil

}

func (d *driver) GetVolumeMapping() ([]*core.BlockDevice, error) {
	blockDevices, err := d.getBlockDevices()
	if err != nil {
		return nil,
			goof.WithFieldsE(ef(), "error getting block devices", err)
	}

	var BlockDevices []*core.BlockDevice
	for _, blockDevice := range blockDevices {
		sdBlockDevice := &core.BlockDevice{
			ProviderName: providerName,
			InstanceID:   d.sdc.Sdc.ID,
			Region:       blockDevice.MdmID,
			DeviceName:   blockDevice.SdcDevice,
			VolumeID:     blockDevice.VolumeID,
			Status:       "",
		}
		BlockDevices = append(BlockDevices, sdBlockDevice)
	}

	log.WithFields(log.Fields{
		"provider":     providerName,
		"blockDevices": BlockDevices,
	}).Debug("got block device mappings")
	return BlockDevices, nil
}

// 	// VolumeDetach detaches a volume.
func (d *driver) VolumeDetach(
	ctx types.Context,
	volumeID string,
	opts *types.VolumeDetachOpts) (*types.Volume, error) {
	fields := eff(map[string]interface{}{
		"moduleName": ctx,
		"volumeId":   volumeID,
	})

	if volumeID == "" {
		return nil, goof.WithFields(fields, "volumeId is required for VolumeDetach")
	}
	vols, err := d.getVolume(ctx, volumeID, "", types.VolAttReqTrue)
	if err != nil {
		return nil, err
	}

	resp := volumeattach.Delete(
		d.client, vols[0].Attachments[0].InstanceID.ID, volumeID)
	if resp.Err != nil {
		return nil, goof.WithFieldsE(fields, "error detaching volume", resp.Err)
	}
	ctx.WithFields(fields).Debug("waiting for volume to detach")
	volume, err := d.waitVolumeAttachStatus(ctx, volumeID, false)
	if err == nil {
		return volume, nil
	}
	log.WithFields(fields).Debug("volume detached")
	return nil, nil
}

func (d *driver) waitVolumeAttachStatus(
	ctx types.Context,
	volumeID string,
	attachmentNeeded bool) (*types.Volume, error) {
	fields := eff(map[string]interface{}{
		"moduleName": ctx,
		"volumeId":   volumeID,
	})

	if volumeID == "" {
		return nil, goof.WithFields(fields, "volumeId is required")
	}

	for {
		volume, err := d.VolumeInspect(
			ctx, volumeID,
			&types.VolumeInspectOpts{Attachments: types.VolAttReqTrue})
		if err != nil {
			return nil, goof.WithFieldsE(fields, "error getting volume when waiting", err)
		}

		if attachmentNeeded {
			if len(volume.Attachments) > 0 {
				return volume, nil
			}
		} else {
			if len(volume.Attachments) == 0 {
				return volume, nil
			}
		}
		time.Sleep(1 * time.Second)
	}
}

func (d *driver) GetVolumeMapping() ([]*core.BlockDevice, error) {
	blockDevices, err := d.getBlockDevices(d.instanceID)
	if err != nil {
		return nil,
			goof.WithFieldsE(eff(goof.Fields{
				"instanceId": d.instanceID,
			}), "error getting block devices", err)
	}

	var BlockDevices []*core.BlockDevice
	for _, blockDevice := range blockDevices {
		sdBlockDevice := &core.BlockDevice{
			ProviderName: providerName,
			InstanceID:   d.instanceID,
			VolumeID:     blockDevice.VolumeID,
			DeviceName:   blockDevice.Device,
			Region:       d.region,
			Status:       "",
		}
		BlockDevices = append(BlockDevices, sdBlockDevice)
	}

	return BlockDevices, nil

}