func createPortForward(
	d *schema.ResourceData,
	meta interface{},
	forward map[string]interface{}) error {
	cs := meta.(*cloudstack.CloudStackClient)

	// Make sure all required parameters are there
	if err := verifyPortForwardParams(d, forward); err != nil {
		return err
	}

	virtualmachine, ok := forward["virtual_machine_id"]
	if !ok {
		virtualmachine, ok = forward["virtual_machine"]
	}
	if !ok {
		return errors.New(
			"Either `virtual_machine_id` or [deprecated] `virtual_machine` must be provided.")
	}

	// Retrieve the virtual_machine ID
	virtualmachineid, e := retrieveID(
		cs,
		"virtual_machine",
		virtualmachine.(string),
		cloudstack.WithProject(d.Get("project").(string)),
	)
	if e != nil {
		return e.Error()
	}

	vm, _, err := cs.VirtualMachine.GetVirtualMachineByID(
		virtualmachineid,
		cloudstack.WithProject(d.Get("project").(string)),
	)
	if err != nil {
		return err
	}

	// Create a new parameter struct
	p := cs.Firewall.NewCreatePortForwardingRuleParams(d.Id(), forward["private_port"].(int),
		forward["protocol"].(string), forward["public_port"].(int), vm.Id)

	// Set the network ID of the default network, needed when public IP address
	// is not associated with any Guest network yet (VPC case)
	p.SetNetworkid(vm.Nic[0].Networkid)

	// Do not open the firewall automatically in any case
	p.SetOpenfirewall(false)

	r, err := cs.Firewall.CreatePortForwardingRule(p)
	if err != nil {
		return err
	}

	forward["uuid"] = r.Id

	return nil
}

func resourceCloudStackDiskRead(d *schema.ResourceData, meta interface{}) error {
	cs := meta.(*cloudstack.CloudStackClient)

	// Get the volume details
	v, count, err := cs.Volume.GetVolumeByID(
		d.Id(),
		cloudstack.WithProject(d.Get("project").(string)),
	)
	if err != nil {
		if count == 0 {
			d.SetId("")
			return nil
		}

		return err
	}

	d.Set("name", v.Name)
	d.Set("attach", v.Attached != "")           // If attached this contains a timestamp when attached
	d.Set("size", int(v.Size/(1024*1024*1024))) // Needed to get GB's again

	setValueOrID(d, "disk_offering", v.Diskofferingname, v.Diskofferingid)
	setValueOrID(d, "project", v.Project, v.Projectid)
	setValueOrID(d, "zone", v.Zonename, v.Zoneid)

	if v.Attached != "" {
		// Get the virtual machine details
		vm, _, err := cs.VirtualMachine.GetVirtualMachineByID(
			v.Virtualmachineid,
			cloudstack.WithProject(d.Get("project").(string)),
		)
		if err != nil {
			return err
		}

		// Get the guest OS type details
		os, _, err := cs.GuestOS.GetOsTypeByID(vm.Guestosid)
		if err != nil {
			return err
		}

		// Get the guest OS category details
		c, _, err := cs.GuestOS.GetOsCategoryByID(os.Oscategoryid)
		if err != nil {
			return err
		}

		d.Set("device", retrieveDeviceName(v.Deviceid, c.Name))
		setValueOrID(d, "virtual_machine", v.Vmname, v.Virtualmachineid)
	}

	return nil
}

func resourceCloudStackInstanceRead(d *schema.ResourceData, meta interface{}) error {
	cs := meta.(*cloudstack.CloudStackClient)

	// Get the virtual machine details
	vm, count, err := cs.VirtualMachine.GetVirtualMachineByID(
		d.Id(),
		cloudstack.WithProject(d.Get("project").(string)),
	)
	if err != nil {
		if count == 0 {
			log.Printf("[DEBUG] Instance %s does no longer exist", d.Get("name").(string))
			d.SetId("")
			return nil
		}

		return err
	}

	// Update the config
	d.Set("name", vm.Name)
	d.Set("display_name", vm.Displayname)
	d.Set("network_id", vm.Nic[0].Networkid)
	d.Set("ip_address", vm.Nic[0].Ipaddress)
	d.Set("group", vm.Group)

	setValueOrID(d, "service_offering", vm.Serviceofferingname, vm.Serviceofferingid)
	setValueOrID(d, "template", vm.Templatename, vm.Templateid)
	setValueOrID(d, "project", vm.Project, vm.Projectid)
	setValueOrID(d, "zone", vm.Zonename, vm.Zoneid)

	return nil
}

// associatePublicIPAddress associates a new IP and sets the address and it's ID.
func (lb *loadBalancer) associatePublicIPAddress() error {
	glog.V(4).Infof("Allocate new IP for load balancer: %v", lb.name)
	// If a network belongs to a VPC, the IP address needs to be associated with
	// the VPC instead of with the network.
	network, count, err := lb.Network.GetNetworkByID(lb.networkID, cloudstack.WithProject(lb.projectID))
	if err != nil {
		if count == 0 {
			return fmt.Errorf("could not find network %v", lb.networkID)
		}
		return fmt.Errorf("error retrieving network: %v", err)
	}

	p := lb.Address.NewAssociateIpAddressParams()

	if network.Vpcid != "" {
		p.SetVpcid(network.Vpcid)
	} else {
		p.SetNetworkid(lb.networkID)
	}

	if lb.projectID != "" {
		p.SetProjectid(lb.projectID)
	}

	// Associate a new IP address
	r, err := lb.Address.AssociateIpAddress(p)
	if err != nil {
		return fmt.Errorf("error associating new IP address: %v", err)
	}

	lb.ipAddr = r.Ipaddress
	lb.ipAddrID = r.Id

	return nil
}

func resourceCloudStackNetworkACLRead(d *schema.ResourceData, meta interface{}) error {
	cs := meta.(*cloudstack.CloudStackClient)

	// Get the network ACL list details
	f, count, err := cs.NetworkACL.GetNetworkACLListByID(
		d.Id(),
		cloudstack.WithProject(d.Get("project").(string)),
	)
	if err != nil {
		if count == 0 {
			log.Printf(
				"[DEBUG] Network ACL list %s does no longer exist", d.Get("name").(string))
			d.SetId("")
			return nil
		}

		return err
	}

	d.Set("name", f.Name)
	d.Set("description", f.Description)
	d.Set("vpc_id", f.Vpcid)

	return nil
}

func resourceCloudStackStaticNATRead(d *schema.ResourceData, meta interface{}) error {
	cs := meta.(*cloudstack.CloudStackClient)

	// Get the IP address details
	ip, count, err := cs.Address.GetPublicIpAddressByID(
		d.Id(),
		cloudstack.WithProject(d.Get("project").(string)),
	)
	if err != nil {
		if count == 0 {
			log.Printf("[DEBUG] IP address with ID %s no longer exists", d.Id())
			d.SetId("")
			return nil
		}

		return err
	}

	if !ip.Isstaticnat {
		log.Printf("[DEBUG] Static NAT is no longer enabled for IP address with ID %s", d.Id())
		d.SetId("")
		return nil
	}

	d.Set("virtual_machine_id", ip.Virtualmachineid)
	d.Set("vm_guest_ip", ip.Vmipaddress)

	setValueOrID(d, "project", ip.Project, ip.Projectid)

	return nil
}

func resourceCloudStackIPAddressRead(d *schema.ResourceData, meta interface{}) error {
	cs := meta.(*cloudstack.CloudStackClient)

	// Get the IP address details
	ip, count, err := cs.Address.GetPublicIpAddressByID(
		d.Id(),
		cloudstack.WithProject(d.Get("project").(string)),
	)
	if err != nil {
		if count == 0 {
			log.Printf(
				"[DEBUG] IP address with ID %s is no longer associated", d.Id())
			d.SetId("")
			return nil
		}

		return err
	}

	// Updated the IP address
	d.Set("ip_address", ip.Ipaddress)

	if _, ok := d.GetOk("network_id"); ok {
		d.Set("network_id", ip.Associatednetworkid)
	}

	if _, ok := d.GetOk("vpc_id"); ok {
		d.Set("vpc_id", ip.Vpcid)
	}

	setValueOrID(d, "project", ip.Project, ip.Projectid)

	return nil
}

func resourceCloudStackAffinityGroupRead(d *schema.ResourceData, meta interface{}) error {
	cs := meta.(*cloudstack.CloudStackClient)

	log.Printf("[DEBUG] Rerieving affinity group %s", d.Get("name").(string))

	// Get the affinity group details
	ag, count, err := cs.AffinityGroup.GetAffinityGroupByID(
		d.Id(),
		cloudstack.WithProject(d.Get("project").(string)),
	)
	if err != nil {
		if count == 0 {
			log.Printf("[DEBUG] Affinity group %s does not longer exist", d.Get("name").(string))
			d.SetId("")
			return nil
		}

		return err
	}

	// Update the config
	d.Set("name", ag.Name)
	d.Set("description", ag.Description)
	d.Set("type", ag.Type)

	return nil
}

func resourceCloudStackLoadBalancerRuleRead(d *schema.ResourceData, meta interface{}) error {
	cs := meta.(*cloudstack.CloudStackClient)

	// Get the load balancer details
	lb, count, err := cs.LoadBalancer.GetLoadBalancerRuleByID(
		d.Id(),
		cloudstack.WithProject(d.Get("project").(string)),
	)
	if err != nil {
		if count == 0 {
			log.Printf("[DEBUG] Load balancer rule %s does no longer exist", d.Get("name").(string))
			d.SetId("")
			return nil
		}

		return err
	}

	d.Set("algorithm", lb.Algorithm)
	d.Set("public_port", lb.Publicport)
	d.Set("private_port", lb.Privateport)
	d.Set("ip_address_id", lb.Publicipid)

	// Only set network if user specified it to avoid spurious diffs
	if _, ok := d.GetOk("network_id"); ok {
		d.Set("network_id", lb.Networkid)
	}

	setValueOrID(d, "project", lb.Project, lb.Projectid)

	return nil
}

func resourceCloudStackDiskRead(d *schema.ResourceData, meta interface{}) error {
	cs := meta.(*cloudstack.CloudStackClient)

	// Get the volume details
	v, count, err := cs.Volume.GetVolumeByID(
		d.Id(),
		cloudstack.WithProject(d.Get("project").(string)),
	)
	if err != nil {
		if count == 0 {
			d.SetId("")
			return nil
		}

		return err
	}

	d.Set("name", v.Name)
	d.Set("attach", v.Attached != "")           // If attached this contains a timestamp when attached
	d.Set("size", int(v.Size/(1024*1024*1024))) // Needed to get GB's again

	setValueOrID(d, "disk_offering", v.Diskofferingname, v.Diskofferingid)
	setValueOrID(d, "project", v.Project, v.Projectid)
	setValueOrID(d, "zone", v.Zonename, v.Zoneid)

	if v.Attached != "" {
		d.Set("device_id", int(v.Deviceid))
		d.Set("virtual_machine_id", v.Virtualmachineid)
	}

	return nil
}

func resourceCloudStackSecurityGroupRuleRead(d *schema.ResourceData, meta interface{}) error {
	cs := meta.(*cloudstack.CloudStackClient)

	// Get the security group details
	sg, count, err := cs.SecurityGroup.GetSecurityGroupByID(
		d.Id(),
		cloudstack.WithProject(d.Get("project").(string)),
	)
	if err != nil {
		if count == 0 {
			log.Printf("[DEBUG] Security group %s does not longer exist", d.Get("name").(string))
			d.SetId("")
			return nil
		}

		return err
	}

	// Make a map of all the rule indexes so we can easily find a rule
	sgRules := append(sg.Ingressrule, sg.Egressrule...)
	ruleIndex := make(map[string]int, len(sgRules))
	for idx, r := range sgRules {
		ruleIndex[r.Ruleid] = idx
	}

	// Create an empty schema.Set to hold all rules
	rules := resourceCloudStackSecurityGroupRule().Schema["rule"].ZeroValue().(*schema.Set)

	// Read all rules that are configured
	if rs := d.Get("rule").(*schema.Set); rs.Len() > 0 {
		for _, rule := range rs.List() {
			rule := rule.(map[string]interface{})

			// First get any existing values
			cidrList, cidrListOK := rule["cidr_list"].(*schema.Set)
			usgList, usgListOk := rule["user_security_group_list"].(*schema.Set)

			// Then reset the values to a new empty set
			rule["cidr_list"] = &schema.Set{F: schema.HashString}
			rule["user_security_group_list"] = &schema.Set{F: schema.HashString}

			if cidrListOK && cidrList.Len() > 0 {
				for _, cidr := range cidrList.List() {
					readSecurityGroupRule(sg, ruleIndex, rule, cidr.(string))
				}
			}

			if usgListOk && usgList.Len() > 0 {
				for _, usg := range usgList.List() {
					readSecurityGroupRule(sg, ruleIndex, rule, usg.(string))
				}
			}

			rules.Add(rule)
		}
	}

	return nil
}

func resourceCloudStackVPCRead(d *schema.ResourceData, meta interface{}) error {
	cs := meta.(*cloudstack.CloudStackClient)

	// Get the VPC details
	v, count, err := cs.VPC.GetVPCByID(
		d.Id(),
		cloudstack.WithProject(d.Get("project").(string)),
	)
	if err != nil {
		if count == 0 {
			log.Printf(
				"[DEBUG] VPC %s does no longer exist", d.Get("name").(string))
			d.SetId("")
			return nil
		}

		return err
	}

	d.Set("name", v.Name)
	d.Set("display_text", v.Displaytext)
	d.Set("cidr", v.Cidr)
	d.Set("network_domain", v.Networkdomain)

	// Get the VPC offering details
	o, _, err := cs.VPC.GetVPCOfferingByID(v.Vpcofferingid)
	if err != nil {
		return err
	}

	setValueOrID(d, "vpc_offering", o.Name, v.Vpcofferingid)
	setValueOrID(d, "project", v.Project, v.Projectid)
	setValueOrID(d, "zone", v.Zonename, v.Zoneid)

	// Create a new parameter struct
	p := cs.Address.NewListPublicIpAddressesParams()
	p.SetVpcid(d.Id())
	p.SetIssourcenat(true)

	// If there is a project supplied, we retrieve and set the project id
	if err := setProjectid(p, cs, d); err != nil {
		return err
	}

	// Get the source NAT IP assigned to the VPC
	l, err := cs.Address.ListPublicIpAddresses(p)
	if err != nil {
		return err
	}

	if l.Count != 1 {
		return fmt.Errorf("Unexpected number (%d) of source NAT IPs returned", l.Count)
	}

	d.Set("source_nat_ip", l.PublicIpAddresses[0].Ipaddress)

	return nil
}

func createPortForward(d *schema.ResourceData, meta interface{}, forward map[string]interface{}) error {
	cs := meta.(*cloudstack.CloudStackClient)

	// Make sure all required parameters are there
	if err := verifyPortForwardParams(d, forward); err != nil {
		return err
	}

	vm, _, err := cs.VirtualMachine.GetVirtualMachineByID(
		forward["virtual_machine_id"].(string),
		cloudstack.WithProject(d.Get("project").(string)),
	)
	if err != nil {
		return err
	}

	// Create a new parameter struct
	p := cs.Firewall.NewCreatePortForwardingRuleParams(d.Id(), forward["private_port"].(int),
		forward["protocol"].(string), forward["public_port"].(int), vm.Id)

	if vmGuestIP, ok := forward["vm_guest_ip"]; ok {
		p.SetVmguestip(vmGuestIP.(string))

		// Set the network ID based on the guest IP, needed when the public IP address
		// is not associated with any network yet
	NICS:
		for _, nic := range vm.Nic {
			if vmGuestIP.(string) == nic.Ipaddress {
				p.SetNetworkid(nic.Networkid)
				break NICS
			}
			for _, ip := range nic.Secondaryip {
				if vmGuestIP.(string) == ip.Ipaddress {
					p.SetNetworkid(nic.Networkid)
					break NICS
				}
			}
		}
	} else {
		// If no guest IP is configured, use the primary NIC
		p.SetNetworkid(vm.Nic[0].Networkid)
	}

	// Do not open the firewall automatically in any case
	p.SetOpenfirewall(false)

	r, err := cs.Firewall.CreatePortForwardingRule(p)
	if err != nil {
		return err
	}

	forward["uuid"] = r.Id

	return nil
}

func resourceCloudStackDiskDetach(d *schema.ResourceData, meta interface{}) error {
	cs := meta.(*cloudstack.CloudStackClient)

	// Check if the volume is actually attached, before detaching
	if attached, err := isAttached(d, meta); err != nil || !attached {
		return err
	}

	// Create a new parameter struct
	p := cs.Volume.NewDetachVolumeParams()

	// Set the volume ID
	p.SetId(d.Id())

	// Detach the currently attached volume
	if _, err := cs.Volume.DetachVolume(p); err != nil {
		// Retrieve the virtual_machine ID
		virtualmachineid, e := retrieveID(
			cs,
			"virtual_machine",
			d.Get("virtual_machine").(string),
			cloudstack.WithProject(d.Get("project").(string)),
		)
		if e != nil {
			return e.Error()
		}

		// Create a new parameter struct
		pd := cs.VirtualMachine.NewStopVirtualMachineParams(virtualmachineid)

		// Stop the virtual machine in order to be able to detach the disk
		if _, err := cs.VirtualMachine.StopVirtualMachine(pd); err != nil {
			return err
		}

		// Try again to detach the currently attached volume
		if _, err := cs.Volume.DetachVolume(p); err != nil {
			return err
		}

		// Create a new parameter struct
		pu := cs.VirtualMachine.NewStartVirtualMachineParams(virtualmachineid)

		// Start the virtual machine again
		if _, err := cs.VirtualMachine.StartVirtualMachine(pu); err != nil {
			return err
		}
	}

	return nil
}

func isAttached(d *schema.ResourceData, meta interface{}) (bool, error) {
	cs := meta.(*cloudstack.CloudStackClient)

	// Get the volume details
	v, _, err := cs.Volume.GetVolumeByID(
		d.Id(),
		cloudstack.WithProject(d.Get("project").(string)),
	)
	if err != nil {
		return false, err
	}

	return v.Attached != "", nil
}

func resourceCloudStackNetworkRead(d *schema.ResourceData, meta interface{}) error {
	cs := meta.(*cloudstack.CloudStackClient)

	// Get the virtual machine details
	n, count, err := cs.Network.GetNetworkByID(
		d.Id(),
		cloudstack.WithProject(d.Get("project").(string)),
	)
	if err != nil {
		if count == 0 {
			log.Printf(
				"[DEBUG] Network %s does no longer exist", d.Get("name").(string))
			d.SetId("")
			return nil
		}

		return err
	}

	d.Set("name", n.Name)
	d.Set("display_text", n.Displaytext)
	d.Set("cidr", n.Cidr)
	d.Set("gateway", n.Gateway)

	_, vpcID := d.GetOk("vpc_id")
	_, vpc := d.GetOk("vpc")
	if vpcID || vpc {
		d.Set("vpc_id", n.Vpcid)

		// Since we're in a VPC, also update the ACL ID. If we don't
		// have an ACL ID make sure we set the default value instead.
		if n.Aclid == "" {
			n.Aclid = none
		}
		d.Set("acl_id", n.Aclid)
	}

	// Read the tags and store them in a map
	tags := make(map[string]interface{})
	for item := range n.Tags {
		tags[n.Tags[item].Key] = n.Tags[item].Value
	}
	d.Set("tags", tags)

	setValueOrID(d, "network_offering", n.Networkofferingname, n.Networkofferingid)
	setValueOrID(d, "project", n.Project, n.Projectid)
	setValueOrID(d, "zone", n.Zonename, n.Zoneid)

	return nil
}

func resourceCloudStackStaticNATCreate(d *schema.ResourceData, meta interface{}) error {
	cs := meta.(*cloudstack.CloudStackClient)

	ipaddressid := d.Get("ip_address_id").(string)

	vm, _, err := cs.VirtualMachine.GetVirtualMachineByID(
		d.Get("virtual_machine_id").(string),
		cloudstack.WithProject(d.Get("project").(string)),
	)
	if err != nil {
		return err
	}

	// Create a new parameter struct
	p := cs.NAT.NewEnableStaticNatParams(ipaddressid, vm.Id)

	if vmGuestIP, ok := d.GetOk("vm_guest_ip"); ok {
		p.SetVmguestip(vmGuestIP.(string))

		// Set the network ID based on the guest IP, needed when the public IP address
		// is not associated with any network yet
	NICS:
		for _, nic := range vm.Nic {
			if vmGuestIP.(string) == nic.Ipaddress {
				p.SetNetworkid(nic.Networkid)
				break NICS
			}
			for _, ip := range nic.Secondaryip {
				if vmGuestIP.(string) == ip.Ipaddress {
					p.SetNetworkid(nic.Networkid)
					break NICS
				}
			}
		}
	} else {
		// If no guest IP is configured, use the primary NIC
		p.SetNetworkid(vm.Nic[0].Networkid)
	}

	_, err = cs.NAT.EnableStaticNat(p)
	if err != nil {
		return fmt.Errorf("Error enabling static NAT: %s", err)
	}

	d.SetId(ipaddressid)

	return resourceCloudStackStaticNATRead(d, meta)
}

func resourceCloudStackDiskAttach(d *schema.ResourceData, meta interface{}) error {
	cs := meta.(*cloudstack.CloudStackClient)

	// First check if the disk isn't already attached
	if attached, err := isAttached(d, meta); err != nil || attached {
		return err
	}

	// Retrieve the virtual_machine ID
	virtualmachineid, e := retrieveID(
		cs,
		"virtual_machine",
		d.Get("virtual_machine").(string),
		cloudstack.WithProject(d.Get("project").(string)),
	)
	if e != nil {
		return e.Error()
	}

	// Create a new parameter struct
	p := cs.Volume.NewAttachVolumeParams(d.Id(), virtualmachineid)

	if device, ok := d.GetOk("device"); ok {
		// Retrieve the device ID
		deviceid := retrieveDeviceID(device.(string))
		if deviceid == -1 {
			return fmt.Errorf("Device %s is not a valid device", device.(string))
		}

		// Set the device ID
		p.SetDeviceid(deviceid)
	}

	// Attach the new volume
	r, err := Retry(4, retryableAttachVolumeFunc(cs, p))
	if err != nil {
		return err
	}

	d.SetId(r.(*cloudstack.AttachVolumeResponse).Id)

	return nil
}

func resourceCloudStackStaticNATExists(d *schema.ResourceData, meta interface{}) (bool, error) {
	cs := meta.(*cloudstack.CloudStackClient)

	// Get the IP address details
	ip, count, err := cs.Address.GetPublicIpAddressByID(
		d.Id(),
		cloudstack.WithProject(d.Get("project").(string)),
	)
	if err != nil {
		if count == 0 {
			log.Printf("[DEBUG] IP address with ID %s no longer exists", d.Id())
			return false, nil
		}

		return false, err
	}

	return ip.Isstaticnat, nil
}

func resourceCloudStackPortForwardCreate(d *schema.ResourceData, meta interface{}) error {
	cs := meta.(*cloudstack.CloudStackClient)

	ipaddress, ok := d.GetOk("ip_address_id")
	if !ok {
		ipaddress, ok = d.GetOk("ipaddress")
	}
	if !ok {
		return errors.New("Either `ip_address_id` or [deprecated] `ipaddress` must be provided.")
	}

	// Retrieve the ipaddress ID
	ipaddressid, e := retrieveID(
		cs,
		"ip_address",
		ipaddress.(string),
		cloudstack.WithProject(d.Get("project").(string)),
	)
	if e != nil {
		return e.Error()
	}

	// We need to set this upfront in order to be able to save a partial state
	d.SetId(ipaddressid)

	// Create all forwards that are configured
	if nrs := d.Get("forward").(*schema.Set); nrs.Len() > 0 {
		// Create an empty schema.Set to hold all forwards
		forwards := resourceCloudStackPortForward().Schema["forward"].ZeroValue().(*schema.Set)

		err := createPortForwards(d, meta, forwards, nrs)

		// We need to update this first to preserve the correct state
		d.Set("forward", forwards)

		if err != nil {
			return err
		}
	}

	return resourceCloudStackPortForwardRead(d, meta)
}