// deleteInstanceNetworkSecurityRules deletes network security rules in the
// internal network security group that correspond to the specified machine.
//
// This is expected to delete *all* security rules related to the instance,
// i.e. both the ones opened by OpenPorts above, and the ones opened for API
// access.
func deleteInstanceNetworkSecurityRules(
	resourceGroup string, id instance.Id,
	nsgClient network.SecurityGroupsClient,
	securityRuleClient network.SecurityRulesClient,
) error {
	nsg, err := nsgClient.Get(resourceGroup, internalSecurityGroupName)
	if err != nil {
		return errors.Annotate(err, "querying network security group")
	}
	if nsg.Properties.SecurityRules == nil {
		return nil
	}
	prefix := instanceNetworkSecurityRulePrefix(id)
	for _, rule := range *nsg.Properties.SecurityRules {
		ruleName := to.String(rule.Name)
		if !strings.HasPrefix(ruleName, prefix) {
			continue
		}
		result, err := securityRuleClient.Delete(
			resourceGroup,
			internalSecurityGroupName,
			ruleName,
		)
		if err != nil {
			if result.Response == nil || result.StatusCode != http.StatusNotFound {
				return errors.Annotatef(err, "deleting security rule %q", ruleName)
			}
		}
	}
	return nil
}

// deleteInstanceNetworkSecurityRules deletes network security rules in the
// internal network security group that correspond to the specified machine.
//
// This is expected to delete *all* security rules related to the instance,
// i.e. both the ones opened by OpenPorts above, and the ones opened for API
// access.
func deleteInstanceNetworkSecurityRules(
	resourceGroup string, id instance.Id,
	nsgClient network.SecurityGroupsClient,
	securityRuleClient network.SecurityRulesClient,
	callAPI callAPIFunc,
) error {
	var nsg network.SecurityGroup
	if err := callAPI(func() (autorest.Response, error) {
		var err error
		nsg, err = nsgClient.Get(resourceGroup, internalSecurityGroupName, "")
		return nsg.Response, err
	}); err != nil {
		return errors.Annotate(err, "querying network security group")
	}
	if nsg.Properties.SecurityRules == nil {
		return nil
	}
	prefix := instanceNetworkSecurityRulePrefix(id)
	for _, rule := range *nsg.Properties.SecurityRules {
		ruleName := to.String(rule.Name)
		if !strings.HasPrefix(ruleName, prefix) {
			continue
		}
		var result autorest.Response
		err := callAPI(func() (autorest.Response, error) {
			var err error
			result, err = securityRuleClient.Delete(
				resourceGroup,
				internalSecurityGroupName,
				ruleName,
				nil, // abort channel
			)
			return result, err
		})
		if err != nil {
			if result.Response == nil || result.StatusCode != http.StatusNotFound {
				return errors.Annotatef(err, "deleting security rule %q", ruleName)
			}
		}
	}
	return nil
}

func newNetworkProfile(
	client network.ManagementClient,
	vmName string,
	apiPort *int,
	internalSubnet *network.Subnet,
	resourceGroup string,
	location string,
	tags map[string]string,
) (*compute.NetworkProfile, error) {
	logger.Debugf("creating network profile for %q", vmName)

	// Create a public IP for the NIC. Public IP addresses are dynamic.
	logger.Debugf("- allocating public IP address")
	pipClient := network.PublicIPAddressesClient{client}
	publicIPAddressParams := network.PublicIPAddress{
		Location: to.StringPtr(location),
		Tags:     toTagsPtr(tags),
		Properties: &network.PublicIPAddressPropertiesFormat{
			PublicIPAllocationMethod: network.Dynamic,
		},
	}
	publicIPAddressName := vmName + "-public-ip"
	publicIPAddress, err := pipClient.CreateOrUpdate(resourceGroup, publicIPAddressName, publicIPAddressParams)
	if err != nil {
		return nil, errors.Annotatef(err, "creating public IP address for %q", vmName)
	}

	// Determine the next available private IP address.
	nicClient := network.InterfacesClient{client}
	privateIPAddress, err := nextSubnetIPAddress(nicClient, resourceGroup, internalSubnet)
	if err != nil {
		return nil, errors.Annotatef(err, "querying private IP addresses")
	}

	// Create a primary NIC for the machine. This needs to be static, so
	// that we can create security rules that don't become invalid.
	logger.Debugf("- creating primary NIC")
	ipConfigurations := []network.InterfaceIPConfiguration{{
		Name: to.StringPtr("primary"),
		Properties: &network.InterfaceIPConfigurationPropertiesFormat{
			PrivateIPAddress:          to.StringPtr(privateIPAddress),
			PrivateIPAllocationMethod: network.Static,
			Subnet:          &network.SubResource{internalSubnet.ID},
			PublicIPAddress: &network.SubResource{publicIPAddress.ID},
		},
	}}
	primaryNicName := vmName + "-primary"
	primaryNicParams := network.Interface{
		Location: to.StringPtr(location),
		Tags:     toTagsPtr(tags),
		Properties: &network.InterfacePropertiesFormat{
			IPConfigurations: &ipConfigurations,
		},
	}
	primaryNic, err := nicClient.CreateOrUpdate(resourceGroup, primaryNicName, primaryNicParams)
	if err != nil {
		return nil, errors.Annotatef(err, "creating network interface for %q", vmName)
	}

	// Create a network security rule for the machine if we need to open
	// the API server port.
	if apiPort != nil {
		logger.Debugf("- querying network security group")
		securityGroupClient := network.SecurityGroupsClient{client}
		securityGroupName := internalSecurityGroupName
		securityGroup, err := securityGroupClient.Get(resourceGroup, securityGroupName)
		if err != nil {
			return nil, errors.Annotate(err, "querying network security group")
		}

		// NOTE(axw) this looks like TOCTTOU race territory, but it's
		// safe because we only allocate/deallocate rules in this
		// range during machine (de)provisioning, which is managed by
		// a single goroutine. Non-internal ports are managed by the
		// firewaller exclusively.
		nextPriority, err := nextSecurityRulePriority(
			securityGroup,
			securityRuleInternalSSHInbound+1,
			securityRuleInternalMax,
		)
		if err != nil {
			return nil, errors.Trace(err)
		}

		apiSecurityRuleName := fmt.Sprintf("%s-api", vmName)
		apiSecurityRule := network.SecurityRule{
			Name: to.StringPtr(apiSecurityRuleName),
			Properties: &network.SecurityRulePropertiesFormat{
				Description:              to.StringPtr("Allow API access to server machines"),
				Protocol:                 network.SecurityRuleProtocolTCP,
				SourceAddressPrefix:      to.StringPtr("*"),
				SourcePortRange:          to.StringPtr("*"),
				DestinationAddressPrefix: to.StringPtr(privateIPAddress),
				DestinationPortRange:     to.StringPtr(fmt.Sprint(*apiPort)),
				Access:                   network.Allow,
				Priority:                 to.IntPtr(nextPriority),
				Direction:                network.Inbound,
			},
		}
		logger.Debugf("- creating API network security rule")
//.........这里部分代码省略.........

// createInternalSubnet creates an internal subnet for the specified resource group,
// within the specified virtual network.
//
// NOTE(axw) this method expects an up-to-date VirtualNetwork, and expects that are
// no concurrent subnet additions to the virtual network. At the moment we have only
// three places where we modify subnets: at bootstrap, when a new environment is
// created, and when an environment is destroyed.
func createInternalSubnet(
	client network.ManagementClient,
	resourceGroup string,
	vnet *network.VirtualNetwork,
	location string,
	tags map[string]string,
) (*network.Subnet, error) {

	nextAddressPrefix := (*vnet.Properties.AddressSpace.AddressPrefixes)[0]
	if vnet.Properties.Subnets != nil {
		if len(*vnet.Properties.Subnets) == len(*vnet.Properties.AddressSpace.AddressPrefixes) {
			return nil, errors.Errorf(
				"no available address prefixes in vnet %q",
				to.String(vnet.Name),
			)
		}
		addressPrefixesInUse := make(set.Strings)
		for _, subnet := range *vnet.Properties.Subnets {
			addressPrefixesInUse.Add(to.String(subnet.Properties.AddressPrefix))
		}
		for _, addressPrefix := range *vnet.Properties.AddressSpace.AddressPrefixes {
			if !addressPrefixesInUse.Contains(addressPrefix) {
				nextAddressPrefix = addressPrefix
				break
			}
		}
	}

	// Create a network security group for the environment. There is only
	// one NSG per environment (there's a limit of 100 per subscription),
	// in which we manage rules for each exposed machine.
	securityRules := []network.SecurityRule{sshSecurityRule}
	securityGroupParams := network.SecurityGroup{
		Location: to.StringPtr(location),
		Tags:     toTagsPtr(tags),
		Properties: &network.SecurityGroupPropertiesFormat{
			SecurityRules: &securityRules,
		},
	}
	securityGroupClient := network.SecurityGroupsClient{client}
	securityGroupName := internalSecurityGroupName
	logger.Debugf("creating security group %q", securityGroupName)
	nsg, err := securityGroupClient.CreateOrUpdate(
		resourceGroup, securityGroupName, securityGroupParams,
	)
	if err != nil {
		return nil, errors.Annotatef(err, "creating security group %q", securityGroupName)
	}

	// Now create a subnet with the next available address prefix, and
	// associate the subnet with the NSG created above.
	subnetName := internalSubnetName
	subnetParams := network.Subnet{
		Properties: &network.SubnetPropertiesFormat{
			AddressPrefix:        to.StringPtr(nextAddressPrefix),
			NetworkSecurityGroup: &network.SubResource{nsg.ID},
		},
	}
	logger.Debugf("creating subnet %q (%s)", subnetName, nextAddressPrefix)
	subnetClient := network.SubnetsClient{client}
	subnet, err := subnetClient.CreateOrUpdate(
		resourceGroup, internalNetworkName, subnetName, subnetParams,
	)
	if err != nil {
		return nil, errors.Annotatef(err, "creating subnet %q", subnetName)
	}
	return &subnet, nil
}

// Ports is specified in the Instance interface.
func (inst *azureInstance) Ports(machineId string) (ports []jujunetwork.PortRange, err error) {
	inst.env.mu.Lock()
	nsgClient := network.SecurityGroupsClient{inst.env.network}
	inst.env.mu.Unlock()

	securityGroupName := internalSecurityGroupName
	nsg, err := nsgClient.Get(inst.env.resourceGroup, securityGroupName)
	if err != nil {
		return nil, errors.Annotate(err, "querying network security group")
	}
	if nsg.Properties.SecurityRules == nil {
		return nil, nil
	}

	vmName := resourceName(names.NewMachineTag(machineId))
	prefix := instanceNetworkSecurityRulePrefix(instance.Id(vmName))
	for _, rule := range *nsg.Properties.SecurityRules {
		if rule.Properties.Direction != network.Inbound {
			continue
		}
		if rule.Properties.Access != network.Allow {
			continue
		}
		if to.Int(rule.Properties.Priority) <= securityRuleInternalMax {
			continue
		}
		if !strings.HasPrefix(to.String(rule.Name), prefix) {
			continue
		}

		var portRange jujunetwork.PortRange
		if *rule.Properties.DestinationPortRange == "*" {
			portRange.FromPort = 0
			portRange.ToPort = 65535
		} else {
			portRange, err = jujunetwork.ParsePortRange(
				*rule.Properties.DestinationPortRange,
			)
			if err != nil {
				return nil, errors.Annotatef(
					err, "parsing port range for security rule %q",
					to.String(rule.Name),
				)
			}
		}

		var protocols []string
		switch rule.Properties.Protocol {
		case network.SecurityRuleProtocolTCP:
			protocols = []string{"tcp"}
		case network.SecurityRuleProtocolUDP:
			protocols = []string{"udp"}
		default:
			protocols = []string{"tcp", "udp"}
		}
		for _, protocol := range protocols {
			portRange.Protocol = protocol
			ports = append(ports, portRange)
		}
	}
	return ports, nil
}

// OpenPorts is specified in the Instance interface.
func (inst *azureInstance) OpenPorts(machineId string, ports []jujunetwork.PortRange) error {
	inst.env.mu.Lock()
	nsgClient := network.SecurityGroupsClient{inst.env.network}
	securityRuleClient := network.SecurityRulesClient{inst.env.network}
	inst.env.mu.Unlock()
	internalNetworkAddress, err := inst.internalNetworkAddress()
	if err != nil {
		return errors.Trace(err)
	}

	securityGroupName := internalSecurityGroupName
	nsg, err := nsgClient.Get(inst.env.resourceGroup, securityGroupName)
	if err != nil {
		return errors.Annotate(err, "querying network security group")
	}

	var securityRules []network.SecurityRule
	if nsg.Properties.SecurityRules != nil {
		securityRules = *nsg.Properties.SecurityRules
	} else {
		nsg.Properties.SecurityRules = &securityRules
	}

	// Create rules one at a time; this is necessary to avoid trampling
	// on changes made by the provisioner. We still record rules in the
	// NSG in memory, so we can easily tell which priorities are available.
	vmName := resourceName(names.NewMachineTag(machineId))
	prefix := instanceNetworkSecurityRulePrefix(instance.Id(vmName))
	for _, ports := range ports {
		ruleName := securityRuleName(prefix, ports)

		// Check if the rule already exists; OpenPorts must be idempotent.
		var found bool
		for _, rule := range securityRules {
			if to.String(rule.Name) == ruleName {
				found = true
				break
			}
		}
		if found {
			logger.Debugf("security rule %q already exists", ruleName)
			continue
		}
		logger.Debugf("creating security rule %q", ruleName)

		priority, err := nextSecurityRulePriority(nsg, securityRuleInternalMax+1, securityRuleMax)
		if err != nil {
			return errors.Annotatef(err, "getting security rule priority for %s", ports)
		}

		var protocol network.SecurityRuleProtocol
		switch ports.Protocol {
		case "tcp":
			protocol = network.SecurityRuleProtocolTCP
		case "udp":
			protocol = network.SecurityRuleProtocolUDP
		default:
			return errors.Errorf("invalid protocol %q", ports.Protocol)
		}

		var portRange string
		if ports.FromPort != ports.ToPort {
			portRange = fmt.Sprintf("%d-%d", ports.FromPort, ports.ToPort)
		} else {
			portRange = fmt.Sprint(ports.FromPort)
		}

		rule := network.SecurityRule{
			Properties: &network.SecurityRulePropertiesFormat{
				Description:              to.StringPtr(ports.String()),
				Protocol:                 protocol,
				SourcePortRange:          to.StringPtr("*"),
				DestinationPortRange:     to.StringPtr(portRange),
				SourceAddressPrefix:      to.StringPtr("*"),
				DestinationAddressPrefix: to.StringPtr(internalNetworkAddress.Value),
				Access:    network.Allow,
				Priority:  to.IntPtr(priority),
				Direction: network.Inbound,
			},
		}
		if _, err := securityRuleClient.CreateOrUpdate(
			inst.env.resourceGroup, securityGroupName, ruleName, rule,
		); err != nil {
			return errors.Annotatef(err, "creating security rule for %s", ports)
		}
		securityRules = append(securityRules, rule)
	}
	return nil
}

// createInternalSubnet creates an internal subnet for the specified resource group,
// within the specified virtual network.
//
// Subnets are tied to the resource group of the virtual network, so we must create
// them all in the controller resource group. We create the network security group
// for the subnet in the environment's resource group.
//
// NOTE(axw) this method expects an up-to-date VirtualNetwork, and expects that are
// no concurrent subnet additions to the virtual network. At the moment we have only
// three places where we modify subnets: at bootstrap, when a new environment is
// created, and when an environment is destroyed.
func createInternalSubnet(
	client network.ManagementClient,
	resourceGroup, controllerResourceGroup string,
	vnet *network.VirtualNetwork,
	location string,
	tags map[string]string,
) (*network.Subnet, error) {

	nextAddressPrefix := (*vnet.Properties.AddressSpace.AddressPrefixes)[0]
	if vnet.Properties.Subnets != nil {
		if len(*vnet.Properties.Subnets) == len(*vnet.Properties.AddressSpace.AddressPrefixes) {
			return nil, errors.Errorf(
				"no available address prefixes in vnet %q",
				to.String(vnet.Name),
			)
		}
		addressPrefixesInUse := make(set.Strings)
		for _, subnet := range *vnet.Properties.Subnets {
			addressPrefixesInUse.Add(to.String(subnet.Properties.AddressPrefix))
		}
		for _, addressPrefix := range *vnet.Properties.AddressSpace.AddressPrefixes {
			if !addressPrefixesInUse.Contains(addressPrefix) {
				nextAddressPrefix = addressPrefix
				break
			}
		}
	}

	// Create a network security group for the environment. There is only
	// one NSG per environment (there's a limit of 100 per subscription),
	// in which we manage rules for each exposed machine.
	securityRules := []network.SecurityRule{sshSecurityRule}
	securityGroupParams := network.SecurityGroup{
		Location: to.StringPtr(location),
		Tags:     toTagsPtr(tags),
		Properties: &network.SecurityGroupPropertiesFormat{
			SecurityRules: &securityRules,
		},
	}
	securityGroupClient := network.SecurityGroupsClient{client}
	securityGroupName := internalSecurityGroupName
	logger.Debugf("creating security group %q", securityGroupName)
	_, err := securityGroupClient.CreateOrUpdate(
		resourceGroup, securityGroupName, securityGroupParams,
	)
	if err != nil {
		return nil, errors.Annotatef(err, "creating security group %q", securityGroupName)
	}

	// Now create a subnet with the next available address prefix. The
	// subnet must be created in the controller resource group, as it
	// must be co-located with the vnet.
	subnetName := resourceGroup
	subnetParams := network.Subnet{
		Properties: &network.SubnetPropertiesFormat{
			AddressPrefix: to.StringPtr(nextAddressPrefix),
			// NOTE(axw) we do NOT want to set the network security
			// group as default for the subnet, because that will
			// create a dependency from the controller resource
			// group to environment resource groups. Instead, we
			// set the NSG on NICs.
		},
	}
	logger.Debugf("creating subnet %q (%s)", subnetName, nextAddressPrefix)
	subnetClient := network.SubnetsClient{client}
	subnet, err := subnetClient.CreateOrUpdate(
		controllerResourceGroup, internalNetworkName, subnetName, subnetParams,
	)
	if err != nil {
		return nil, errors.Annotatef(err, "creating subnet %q", subnetName)
	}
	return &subnet, nil
}