ensureBackendPoolDeletedFromNode

// ensureBackendPoolDeletedFromNode ensures the loadBalancer backendAddressPools deleted
// from the specified node, which returns (resourceGroup, vmssName, instanceID, vmssVM, error).
func (ss *scaleSet) ensureBackendPoolDeletedFromNode(nodeName, backendPoolID string) (string, string, string, *compute.VirtualMachineScaleSetVM, error) {
	ssName, instanceID, vm, err := ss.getVmssVM(nodeName, azcache.CacheReadTypeDefault)
	if err != nil {
		if errors.Is(err, cloudprovider.InstanceNotFound) {
			klog.Infof("ensureBackendPoolDeletedFromNode: skipping node %s because it is not found", nodeName)
			return "", "", "", nil, nil
		}

		return "", "", "", nil, err
	}

	// Find primary network interface configuration.
	if vm.NetworkProfileConfiguration.NetworkInterfaceConfigurations == nil {
		klog.V(4).Infof("EnsureHostInPool: cannot obtain the primary network interface configuration, of vm %s, probably because the vm's being deleted", nodeName)
		return "", "", "", nil, nil
	}
	networkInterfaceConfigurations := *vm.NetworkProfileConfiguration.NetworkInterfaceConfigurations
	primaryNetworkInterfaceConfiguration, err := ss.getPrimaryNetworkInterfaceConfiguration(networkInterfaceConfigurations, nodeName)
	if err != nil {
		return "", "", "", nil, err
	}

	// Find primary IP configuration.
	primaryIPConfiguration, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNetworkInterfaceConfiguration)
	if err != nil {
		return "", "", "", nil, err
	}
	if primaryIPConfiguration.LoadBalancerBackendAddressPools == nil || len(*primaryIPConfiguration.LoadBalancerBackendAddressPools) == 0 {
		return "", "", "", nil, nil
	}

	// Construct new loadBalancerBackendAddressPools and remove backendAddressPools from primary IP configuration.
	existingBackendPools := *primaryIPConfiguration.LoadBalancerBackendAddressPools
	newBackendPools := []compute.SubResource{}
	foundPool := false
	for i := len(existingBackendPools) - 1; i >= 0; i-- {
		curPool := existingBackendPools[i]
		if strings.EqualFold(backendPoolID, *curPool.ID) {
			klog.V(10).Infof("ensureBackendPoolDeletedFromNode gets unwanted backend pool %q for node %s", backendPoolID, nodeName)
			foundPool = true
			newBackendPools = append(existingBackendPools[:i], existingBackendPools[i+1:]...)
		}
	}

	// Pool not found, assume it has been already removed.
	if !foundPool {
		return "", "", "", nil, nil
	}

	// Compose a new vmssVM with added backendPoolID.
	primaryIPConfiguration.LoadBalancerBackendAddressPools = &newBackendPools
	newVM := &compute.VirtualMachineScaleSetVM{
		Location: vm.Location,
		VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{
			HardwareProfile: vm.HardwareProfile,
			NetworkProfileConfiguration: &compute.VirtualMachineScaleSetVMNetworkProfileConfiguration{
				NetworkInterfaceConfigurations: &networkInterfaceConfigurations,
			},
		},
	}

	// Get the node resource group.
	nodeResourceGroup, err := ss.GetNodeResourceGroup(nodeName)
	if err != nil {
		return "", "", "", nil, err
	}

	return nodeResourceGroup, ssName, instanceID, newVM, nil
}


GetNodeNameByIPConfigurationID

// GetNodeNameByIPConfigurationID gets the node name and the VMSS name by IP configuration ID.
func (ss *scaleSet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, string, error) {
	matches := vmssIPConfigurationRE.FindStringSubmatch(ipConfigurationID)
	if len(matches) != 4 {
		if ss.DisableAvailabilitySetNodes {
			return "", "", ErrorNotVmssInstance
		}

		klog.V(4).Infof("Can not extract scale set name from ipConfigurationID (%s), assuming it is managed by availability set", ipConfigurationID)
		name, rg, err := ss.availabilitySet.GetNodeNameByIPConfigurationID(ipConfigurationID)
		if err != nil && !errors.Is(err, cloudprovider.InstanceNotFound) {
			klog.Errorf("GetNodeNameByIPConfigurationID: failed to invoke availabilitySet.GetNodeNameByIPConfigurationID: %s", err.Error())
			return "", "", err
		}
		return name, rg, nil
	}

	resourceGroup := matches[1]
	scaleSetName := matches[2]
	instanceID := matches[3]
	vm, err := ss.getVmssVMByInstanceID(resourceGroup, scaleSetName, instanceID, azcache.CacheReadTypeUnsafe)
	if err != nil {
		klog.Errorf("Unable to find node by ipConfigurationID %s: %v", ipConfigurationID, err)
		return "", "", err
	}

	if vm.OsProfile != nil && vm.OsProfile.ComputerName != nil {
		return strings.ToLower(*vm.OsProfile.ComputerName), scaleSetName, nil
	}

	return "", "", nil
}


func getScaleSetAndResourceGroupNameByIPConfigurationID(ipConfigurationID string) (string, string, error) {
	matches := vmssIPConfigurationRE.FindStringSubmatch(ipConfigurationID)
	if len(matches) != 4 {
		klog.V(4).Infof("Can not extract scale set name from ipConfigurationID (%s), assuming it is managed by availability set", ipConfigurationID)
		return "", "", ErrorNotVmssInstance
	}

	resourceGroup := matches[1]
	scaleSetName := matches[2]
	return scaleSetName, resourceGroup, nil
}


ensureBackendPoolDeletedFromVMSS

func (ss *scaleSet) ensureBackendPoolDeletedFromVMSS(service *v1.Service, backendPoolID, vmSetName string, ipConfigurationIDs []string) error {
	vmssNamesMap := make(map[string]bool)

	// the standard load balancer supports multiple vmss in its backend while the basic sku doesn't
	if ss.useStandardLoadBalancer() {
		for _, ipConfigurationID := range ipConfigurationIDs {
			// in this scenario the vmSetName is an empty string and the name of vmss should be obtained from the provider IDs of nodes
			vmssName, resourceGroupName, err := getScaleSetAndResourceGroupNameByIPConfigurationID(ipConfigurationID)
			if err != nil {
				klog.V(4).Infof("ensureBackendPoolDeletedFromVMSS: found VMAS ipcConfigurationID %s, will skip checking and continue", ipConfigurationID)
				continue
			}
			// only vmsses in the resource group same as it's in azure config are included
			if strings.EqualFold(resourceGroupName, ss.ResourceGroup) {
				vmssNamesMap[vmssName] = true
			}
		}
	} else {
		vmssNamesMap[vmSetName] = true
	}

	vmssUpdaters := make([]func() error, 0, len(vmssNamesMap))
	errors := make([]error, 0, len(vmssNamesMap))
	for vmssName := range vmssNamesMap {
		vmssName := vmssName
		vmss, err := ss.getVMSS(vmssName, azcache.CacheReadTypeDefault)
		if err != nil {
			klog.Errorf("ensureBackendPoolDeletedFromVMSS: failed to get VMSS %s: %v", vmssName, err)
			errors = append(errors, err)
			continue
		}

		// When vmss is being deleted, CreateOrUpdate API would report "the vmss is being deleted" error.
		// Since it is being deleted, we shouldn't send more CreateOrUpdate requests for it.
		if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, virtualMachineScaleSetsDeallocating) {
			klog.V(3).Infof("ensureBackendPoolDeletedFromVMSS: found vmss %s being deleted, skipping", vmssName)
			continue
		}
		if vmss.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations == nil {
			klog.V(4).Infof("ensureBackendPoolDeletedFromVMSS: cannot obtain the primary network interface configuration, of vmss %s", vmssName)
			continue
		}
		vmssNIC := *vmss.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations
		primaryNIC, err := ss.getPrimaryNetworkInterfaceConfigurationForScaleSet(vmssNIC, vmssName)
		if err != nil {
			klog.Errorf("ensureBackendPoolDeletedFromVMSS: failed to get the primary network interface config of the VMSS %s: %v", vmssName, err)
			errors = append(errors, err)
			continue
		}
		primaryIPConfig, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNIC)
		if err != nil {
			klog.Errorf("ensureBackendPoolDeletedFromVMSS: failed to the primary IP config from the VMSS %s's network config : %v", vmssName, err)
			errors = append(errors, err)
			continue
		}
		loadBalancerBackendAddressPools := []compute.SubResource{}
		if primaryIPConfig.LoadBalancerBackendAddressPools != nil {
			loadBalancerBackendAddressPools = *primaryIPConfig.LoadBalancerBackendAddressPools
		}

		var found bool
		var newBackendPools []compute.SubResource
		for i := len(loadBalancerBackendAddressPools) - 1; i >= 0; i-- {
			curPool := loadBalancerBackendAddressPools[i]
			if strings.EqualFold(backendPoolID, *curPool.ID) {
				klog.V(10).Infof("ensureBackendPoolDeletedFromVMSS gets unwanted backend pool %q for VMSS %s", backendPoolID, vmssName)
				found = true
				newBackendPools = append(loadBalancerBackendAddressPools[:i], loadBalancerBackendAddressPools[i+1:]...)
			}
		}
		if !found {
			continue
		}

		vmssUpdaters = append(vmssUpdaters, func() error {
			// Compose a new vmss with added backendPoolID.
			primaryIPConfig.LoadBalancerBackendAddressPools = &newBackendPools
			newVMSS := compute.VirtualMachineScaleSet{
				Location: vmss.Location,
				VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
					VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{
						NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{
							NetworkInterfaceConfigurations: &vmssNIC,
						},
					},
				},
			}

			klog.V(2).Infof("ensureBackendPoolDeletedFromVMSS begins to update vmss(%s) with backendPoolID %s", vmssName, backendPoolID)
			rerr := ss.CreateOrUpdateVMSS(ss.ResourceGroup, vmssName, newVMSS)
			if rerr != nil {
				klog.Errorf("ensureBackendPoolDeletedFromVMSS CreateOrUpdateVMSS(%s) with new backendPoolID %s, err: %v", vmssName, backendPoolID, rerr)
				return rerr.Error()
			}

			return nil
		})
	}

	errs := utilerrors.AggregateGoroutines(vmssUpdaters...)
	if errs != nil {
		return utilerrors.Flatten(errs)
	}
	// Fail if there are other errors.
	if len(errors) > 0 {
		return utilerrors.Flatten(utilerrors.NewAggregate(errors))
	}

	return nil
}


EnsureBackendPoolDeleted

// EnsureBackendPoolDeleted ensures the loadBalancer backendAddressPools deleted from the specified nodes.
func (ss *scaleSet) EnsureBackendPoolDeleted(service *v1.Service, backendPoolID, vmSetName string, backendAddressPools *[]network.BackendAddressPool, deleteFromVMSet bool) error {
	// Returns nil if backend address pools already deleted.
	if backendAddressPools == nil {
		return nil
	}

	mc := metrics.NewMetricContext("services", "vmss_ensure_backend_pool_deleted", ss.ResourceGroup, ss.SubscriptionID, service.Name)
	isOperationSucceeded := false
	defer func() {
		mc.ObserveOperationWithResult(isOperationSucceeded)
	}()

	ipConfigurationIDs := []string{}
	for _, backendPool := range *backendAddressPools {
		if strings.EqualFold(*backendPool.ID, backendPoolID) && backendPool.BackendIPConfigurations != nil {
			for _, ipConf := range *backendPool.BackendIPConfigurations {
				if ipConf.ID == nil {
					continue
				}

				ipConfigurationIDs = append(ipConfigurationIDs, *ipConf.ID)
			}
		}
	}

	hostUpdates := make([]func() error, 0, len(ipConfigurationIDs))
	nodeUpdates := make(map[vmssMetaInfo]map[string]compute.VirtualMachineScaleSetVM)
	allErrs := make([]error, 0)
	for i := range ipConfigurationIDs {
		ipConfigurationID := ipConfigurationIDs[i]

		var scaleSetName string
		var err error
		if scaleSetName, err = extractScaleSetNameByProviderID(ipConfigurationID); err == nil {
			// Only remove nodes belonging to specified vmSet to basic LB backends.
			if !ss.useStandardLoadBalancer() && !strings.EqualFold(scaleSetName, vmSetName) {
				continue
			}
		}

		nodeName, _, err := ss.GetNodeNameByIPConfigurationID(ipConfigurationID)
		if err != nil {
			if err == ErrorNotVmssInstance { // Do nothing for the VMAS nodes.
				continue
			}

			if errors.Is(err, cloudprovider.InstanceNotFound) {
				klog.Infof("EnsureBackendPoolDeleted(%s): skipping ip config %s because the corresponding vmss vm is not found", getServiceName(service), ipConfigurationID)
				continue
			}

			klog.Errorf("Failed to GetNodeNameByIPConfigurationID(%s): %v", ipConfigurationID, err)
			allErrs = append(allErrs, err)
			continue
		}

		nodeResourceGroup, nodeVMSS, nodeInstanceID, nodeVMSSVM, err := ss.ensureBackendPoolDeletedFromNode(nodeName, backendPoolID)
		if err != nil {
			if !errors.Is(err, ErrorNotVmssInstance) { // Do nothing for the VMAS nodes.
				klog.Errorf("EnsureBackendPoolDeleted(%s): backendPoolID(%s) - failed with error %v", getServiceName(service), backendPoolID, err)
				allErrs = append(allErrs, err)
			}
			continue
		}

		// No need to update if nodeVMSSVM is nil.
		if nodeVMSSVM == nil {
			continue
		}

		nodeVMSSMetaInfo := vmssMetaInfo{vmssName: nodeVMSS, resourceGroup: nodeResourceGroup}
		if v, ok := nodeUpdates[nodeVMSSMetaInfo]; ok {
			v[nodeInstanceID] = *nodeVMSSVM
		} else {
			nodeUpdates[nodeVMSSMetaInfo] = map[string]compute.VirtualMachineScaleSetVM{
				nodeInstanceID: *nodeVMSSVM,
			}
		}

		// Invalidate the cache since the VMSS VM would be updated.
		defer ss.deleteCacheForNode(nodeName)
	}

	// Update VMs with best effort that have already been added to nodeUpdates.
	for meta, update := range nodeUpdates {
		// create new instance of meta and update for passing to anonymous function
		meta := meta
		update := update
		hostUpdates = append(hostUpdates, func() error {
			ctx, cancel := getContextWithCancel()
			defer cancel()
			klog.V(2).Infof("EnsureBackendPoolDeleted begins to UpdateVMs for VMSS(%s, %s) with backendPoolID %s", meta.resourceGroup, meta.vmssName, backendPoolID)
			rerr := ss.VirtualMachineScaleSetVMsClient.UpdateVMs(ctx, meta.resourceGroup, meta.vmssName, update, "network_update")
			if rerr != nil {
				klog.Errorf("EnsureBackendPoolDeleted UpdateVMs for VMSS(%s, %s) failed with error %v", meta.resourceGroup, meta.vmssName, rerr.Error())
				return rerr.Error()
			}

			return nil
		})
	}
	errs := utilerrors.AggregateGoroutines(hostUpdates...)
	if errs != nil {
		return utilerrors.Flatten(errs)
	}

	// Fail if there are other errors.
	if len(allErrs) > 0 {
		return utilerrors.Flatten(utilerrors.NewAggregate(allErrs))
	}

	// Ensure the backendPoolID is also deleted on VMSS itself.
	if deleteFromVMSet {
		err := ss.ensureBackendPoolDeletedFromVMSS(service, backendPoolID, vmSetName, ipConfigurationIDs)
		if err != nil {
			return err
		}
	}

	isOperationSucceeded = true
	return nil
}


newVMSSCache

func (ss *scaleSet) newVMSSCache() (*azcache.TimedCache, error) {
	getter := func(key string) (interface{}, error) {
		localCache := &sync.Map{} // [vmssName]*vmssEntry

		allResourceGroups, err := ss.GetResourceGroups()
		if err != nil {
			return nil, err
		}

		for _, resourceGroup := range allResourceGroups.List() {
			allScaleSets, rerr := ss.VirtualMachineScaleSetsClient.List(context.Background(), resourceGroup)
			if rerr != nil {
				klog.Errorf("VirtualMachineScaleSetsClient.List failed: %v", rerr)
				return nil, rerr.Error()
			}

			for i := range allScaleSets {
				scaleSet := allScaleSets[i]
				if scaleSet.Name == nil || *scaleSet.Name == "" {
					klog.Warning("failed to get the name of VMSS")
					continue
				}
				localCache.Store(*scaleSet.Name, &vmssEntry{
					vmss:          &scaleSet,
					resourceGroup: resourceGroup,
					lastUpdate:    time.Now().UTC(),
				})
			}
		}

		return localCache, nil
	}

	if ss.Config.VmssCacheTTLInSeconds == 0 {
		ss.Config.VmssCacheTTLInSeconds = vmssCacheTTLDefaultInSeconds
	}
	return azcache.NewTimedcache(time.Duration(ss.Config.VmssCacheTTLInSeconds)*time.Second, getter)
}


extractVmssVMName

func extractVmssVMName(name string) (string, string, error) {
	split := strings.SplitAfter(name, vmssNameSeparator)
	if len(split) < 2 {
		klog.V(3).Infof("Failed to extract vmssVMName %q", name)
		return "", "", ErrorNotVmssInstance
	}

	ssName := strings.Join(split[0:len(split)-1], "")
	// removing the trailing `vmssNameSeparator` since we used SplitAfter
	ssName = ssName[:len(ssName)-1]
	instanceID := split[len(split)-1]
	return ssName, instanceID, nil
}


getVMSSVMCache

// getVMSSVMCache returns an *azcache.TimedCache and cache key for a VMSS (creating that cache if new).
func (ss *scaleSet) getVMSSVMCache(resourceGroup, vmssName string) (string, *azcache.TimedCache, error) {
	cacheKey := strings.ToLower(fmt.Sprintf("%s/%s", resourceGroup, vmssName))
	if entry, ok := ss.vmssVMCache.Load(cacheKey); ok {
		cache := entry.(*azcache.TimedCache)
		return cacheKey, cache, nil
	}

	cache, err := ss.newVMSSVirtualMachinesCache(resourceGroup, vmssName, cacheKey)
	if err != nil {
		return "", nil, err
	}
	ss.vmssVMCache.Store(cacheKey, cache)
	return cacheKey, cache, nil
}


gcVMSSVMCache

// gcVMSSVMCache delete stale VMSS VMs caches from deleted VMSSes.
func (ss *scaleSet) gcVMSSVMCache() error {
	cached, err := ss.vmssCache.Get(vmssKey, azcache.CacheReadTypeUnsafe)
	if err != nil {
		return err
	}

	vmsses := cached.(*sync.Map)
	removed := map[string]bool{}
	ss.vmssVMCache.Range(func(key, value interface{}) bool {
		cacheKey := key.(string)
		vlistIdx := cacheKey[strings.LastIndex(cacheKey, "/")+1:]
		if _, ok := vmsses.Load(vlistIdx); !ok {
			removed[cacheKey] = true
		}
		return true
	})

	for key := range removed {
		ss.vmssVMCache.Delete(key)
	}

	return nil
}


newVMSSVirtualMachinesCache

// newVMSSVirtualMachinesCache instantiates a new VMs cache for VMs belonging to the provided VMSS.
func (ss *scaleSet) newVMSSVirtualMachinesCache(resourceGroupName, vmssName, cacheKey string) (*azcache.TimedCache, error) {
	vmssVirtualMachinesCacheTTL := time.Duration(ss.Config.VmssVirtualMachinesCacheTTLInSeconds) * time.Second

	getter := func(key string) (interface{}, error) {
		localCache := &sync.Map{} // [nodeName]*vmssVirtualMachinesEntry

		oldCache := make(map[string]vmssVirtualMachinesEntry)

		if vmssCache, ok := ss.vmssVMCache.Load(cacheKey); ok {
			// get old cache before refreshing the cache
			cache := vmssCache.(*azcache.TimedCache)
			entry, exists, err := cache.Store.GetByKey(cacheKey)
			if err != nil {
				return nil, err
			}
			if exists {
				cached := entry.(*azcache.AzureCacheEntry).Data
				if cached != nil {
					virtualMachines := cached.(*sync.Map)
					virtualMachines.Range(func(key, value interface{}) bool {
						oldCache[key.(string)] = *value.(*vmssVirtualMachinesEntry)
						return true
					})
				}
			}
		}

		vms, err := ss.listScaleSetVMs(vmssName, resourceGroupName)
		if err != nil {
			return nil, err
		}

		for i := range vms {
			vm := vms[i]
			if vm.OsProfile == nil || vm.OsProfile.ComputerName == nil {
				klog.Warningf("failed to get computerName for vmssVM (%q)", vmssName)
				continue
			}

			computerName := strings.ToLower(*vm.OsProfile.ComputerName)
			if vm.NetworkProfile == nil || vm.NetworkProfile.NetworkInterfaces == nil {
				klog.Warningf("skip caching vmssVM %s since its network profile hasn't initialized yet (probably still under creating)", computerName)
				continue
			}

			vmssVMCacheEntry := &vmssVirtualMachinesEntry{
				resourceGroup:  resourceGroupName,
				vmssName:       vmssName,
				instanceID:     pointer.StringDeref(vm.InstanceID, ""),
				virtualMachine: &vm,
				lastUpdate:     time.Now().UTC(),
			}
			// set cache entry to nil when the VM is under deleting.
			if vm.VirtualMachineScaleSetVMProperties != nil &&
				strings.EqualFold(pointer.StringDeref(vm.VirtualMachineScaleSetVMProperties.ProvisioningState, ""), string(compute.ProvisioningStateDeleting)) {
				klog.V(4).Infof("VMSS virtualMachine %q is under deleting, setting its cache to nil", computerName)
				vmssVMCacheEntry.virtualMachine = nil
			}
			localCache.Store(computerName, vmssVMCacheEntry)

			delete(oldCache, computerName)
		}

		// add old missing cache data with nil entries to prevent aggressive
		// ARM calls during cache invalidation
		for name, vmEntry := range oldCache {
			// if the nil cache entry has existed for vmssVirtualMachinesCacheTTL in the cache
			// then it should not be added back to the cache
			if vmEntry.virtualMachine == nil && time.Since(vmEntry.lastUpdate) > vmssVirtualMachinesCacheTTL {
				klog.V(5).Infof("ignoring expired entries from old cache for %s", name)
				continue
			}
			lastUpdate := time.Now().UTC()
			if vmEntry.virtualMachine == nil {
				// if this is already a nil entry then keep the time the nil
				// entry was first created, so we can cleanup unwanted entries
				lastUpdate = vmEntry.lastUpdate
			}

			klog.V(5).Infof("adding old entries to new cache for %s", name)
			localCache.Store(name, &vmssVirtualMachinesEntry{
				resourceGroup:  vmEntry.resourceGroup,
				vmssName:       vmEntry.vmssName,
				instanceID:     vmEntry.instanceID,
				virtualMachine: nil,
				lastUpdate:     lastUpdate,
			})
		}

		return localCache, nil
	}

	return azcache.NewTimedcache(vmssVirtualMachinesCacheTTL, getter)
}


deleteCacheForNode

func (ss *scaleSet) deleteCacheForNode(nodeName string) error {
	node, err := ss.getNodeIdentityByNodeName(nodeName, azcache.CacheReadTypeUnsafe)
	if err != nil {
		klog.Errorf("deleteCacheForNode(%s) failed with error: %v", nodeName, err)
		return err
	}

	cacheKey, timedcache, err := ss.getVMSSVMCache(node.resourceGroup, node.vmssName)
	if err != nil {
		klog.Errorf("deleteCacheForNode(%s) failed with error: %v", nodeName, err)
		return err
	}

	vmcache, err := timedcache.Get(cacheKey, azcache.CacheReadTypeUnsafe)
	if err != nil {
		klog.Errorf("deleteCacheForNode(%s) failed with error: %v", nodeName, err)
		return err
	}
	virtualMachines := vmcache.(*sync.Map)
	virtualMachines.Delete(nodeName)

	if err := ss.gcVMSSVMCache(); err != nil {
		klog.Errorf("deleteCacheForNode(%s) failed to gc stale vmss caches: %v", nodeName, err)
	}

	return nil
}


newAvailabilitySetNodesCache

func (ss *scaleSet) newAvailabilitySetNodesCache() (*azcache.TimedCache, error) {
	getter := func(key string) (interface{}, error) {
		vmNames := sets.NewString()
		resourceGroups, err := ss.GetResourceGroups()
		if err != nil {
			return nil, err
		}

		for _, resourceGroup := range resourceGroups.List() {
			vmList, err := ss.Cloud.ListVirtualMachines(resourceGroup)
			if err != nil {
				return nil, err
			}

			for _, vm := range vmList {
				if vm.Name != nil {
					vmNames.Insert(*vm.Name)
				}
			}
		}

		// store all the node names in the cluster when the cache data was created.
		nodeNames, err := ss.GetNodeNames()
		if err != nil {
			return nil, err
		}

		localCache := availabilitySetEntry{
			vmNames:   vmNames,
			nodeNames: nodeNames,
		}

		return localCache, nil
	}

	if ss.Config.AvailabilitySetNodesCacheTTLInSeconds == 0 {
		ss.Config.AvailabilitySetNodesCacheTTLInSeconds = availabilitySetNodesCacheTTLDefaultInSeconds
	}
	return azcache.NewTimedcache(time.Duration(ss.Config.AvailabilitySetNodesCacheTTLInSeconds)*time.Second, getter)
}


isNodeManagedByAvailabilitySet

func (ss *scaleSet) isNodeManagedByAvailabilitySet(nodeName string, crt azcache.AzureCacheReadType) (bool, error) {
	// Assume all nodes are managed by VMSS when DisableAvailabilitySetNodes is enabled.
	if ss.DisableAvailabilitySetNodes {
		klog.V(2).Infof("Assuming node %q is managed by VMSS since DisableAvailabilitySetNodes is set to true", nodeName)
		return false, nil
	}

	cached, err := ss.availabilitySetNodesCache.Get(availabilitySetNodesKey, crt)
	if err != nil {
		return false, err
	}

	cachedNodes := cached.(availabilitySetEntry).nodeNames
	// if the node is not in the cache, assume the node has joined after the last cache refresh and attempt to refresh the cache.
	if !cachedNodes.Has(nodeName) {
		klog.V(2).Infof("Node %s has joined the cluster since the last VM cache refresh, refreshing the cache", nodeName)
		cached, err = ss.availabilitySetNodesCache.Get(availabilitySetNodesKey, azcache.CacheReadTypeForceRefresh)
		if err != nil {
			return false, err
		}
	}

	cachedVMs := cached.(availabilitySetEntry).vmNames
	return cachedVMs.Has(nodeName), nil
}


checkResourceExistsFromError

// checkExistsFromError inspects an error and returns a true if err is nil,
// false if error is an autorest.Error with StatusCode=404 and will return the
// error back if error is another status code or another type of error.
func checkResourceExistsFromError(err *retry.Error) (bool, *retry.Error) {
	if err == nil {
		return true, nil
	}

	if err.HTTPStatusCode == http.StatusNotFound {
		return false, nil
	}

	return false, err
}


getVirtualMachine

// / getVirtualMachine calls 'VirtualMachinesClient.Get' with a timed cache
// / The service side has throttling control that delays responses if there are multiple requests onto certain vm
// / resource request in short period.
func (az *Cloud) getVirtualMachine(nodeName types.NodeName, crt azcache.AzureCacheReadType) (vm compute.VirtualMachine, err error) {
	vmName := string(nodeName)
	cachedVM, err := az.vmCache.Get(vmName, crt)
	if err != nil {
		return vm, err
	}

	if cachedVM == nil {
		return vm, cloudprovider.InstanceNotFound
	}

	return *(cachedVM.(*compute.VirtualMachine)), nil
}


getRouteTable

func (az *Cloud) getRouteTable(crt azcache.AzureCacheReadType) (routeTable network.RouteTable, exists bool, err error) {
	cachedRt, err := az.rtCache.Get(az.RouteTableName, crt)
	if err != nil {
		return routeTable, false, err
	}

	if cachedRt == nil {
		return routeTable, false, nil
	}

	return *(cachedRt.(*network.RouteTable)), true, nil
}


getPublicIPAddress

func (az *Cloud) getPublicIPAddress(pipResourceGroup string, pipName string) (network.PublicIPAddress, bool, error) {
	resourceGroup := az.ResourceGroup
	if pipResourceGroup != "" {
		resourceGroup = pipResourceGroup
	}

	ctx, cancel := getContextWithCancel()
	defer cancel()
	pip, err := az.PublicIPAddressesClient.Get(ctx, resourceGroup, pipName, "")
	exists, rerr := checkResourceExistsFromError(err)
	if rerr != nil {
		return pip, false, rerr.Error()
	}

	if !exists {
		klog.V(2).Infof("Public IP %q not found", pipName)
		return pip, false, nil
	}

	return pip, exists, nil
}


getSubnet

func (az *Cloud) getSubnet(virtualNetworkName string, subnetName string) (network.Subnet, bool, error) {
	var rg string
	if len(az.VnetResourceGroup) > 0 {
		rg = az.VnetResourceGroup
	} else {
		rg = az.ResourceGroup
	}

	ctx, cancel := getContextWithCancel()
	defer cancel()
	subnet, err := az.SubnetsClient.Get(ctx, rg, virtualNetworkName, subnetName, "")
	exists, rerr := checkResourceExistsFromError(err)
	if rerr != nil {
		return subnet, false, rerr.Error()
	}

	if !exists {
		klog.V(2).Infof("Subnet %q not found", subnetName)
		return subnet, false, nil
	}

	return subnet, exists, nil
}


getAzureLoadBalancer

func (az *Cloud) getAzureLoadBalancer(name string, crt azcache.AzureCacheReadType) (lb network.LoadBalancer, exists bool, err error) {
	cachedLB, err := az.lbCache.Get(name, crt)
	if err != nil {
		return lb, false, err
	}

	if cachedLB == nil {
		return lb, false, nil
	}

	return *(cachedLB.(*network.LoadBalancer)), true, nil
}


getSecurityGroup

func (az *Cloud) getSecurityGroup(crt azcache.AzureCacheReadType) (network.SecurityGroup, error) {
	nsg := network.SecurityGroup{}
	if az.SecurityGroupName == "" {
		return nsg, fmt.Errorf("securityGroupName is not configured")
	}

	securityGroup, err := az.nsgCache.Get(az.SecurityGroupName, crt)
	if err != nil {
		return nsg, err
	}

	if securityGroup == nil {
		return nsg, fmt.Errorf("nsg %q not found", az.SecurityGroupName)
	}

	return *(securityGroup.(*network.SecurityGroup)), nil
}


newVMCache

func (az *Cloud) newVMCache() (*azcache.TimedCache, error) {
	getter := func(key string) (interface{}, error) {
		// Currently InstanceView request are used by azure_zones, while the calls come after non-InstanceView
		// request. If we first send an InstanceView request and then a non InstanceView request, the second
		// request will still hit throttling. This is what happens now for cloud controller manager: In this
		// case we do get instance view every time to fulfill the azure_zones requirement without hitting
		// throttling.
		// Consider adding separate parameter for controlling 'InstanceView' once node update issue #56276 is fixed
		ctx, cancel := getContextWithCancel()
		defer cancel()

		resourceGroup, err := az.GetNodeResourceGroup(key)
		if err != nil {
			return nil, err
		}

		vm, verr := az.VirtualMachinesClient.Get(ctx, resourceGroup, key, compute.InstanceView)
		exists, rerr := checkResourceExistsFromError(verr)
		if rerr != nil {
			return nil, rerr.Error()
		}

		if !exists {
			klog.V(2).Infof("Virtual machine %q not found", key)
			return nil, nil
		}

		if vm.VirtualMachineProperties != nil &&
			strings.EqualFold(pointer.StringDeref(vm.VirtualMachineProperties.ProvisioningState, ""), string(compute.ProvisioningStateDeleting)) {
			klog.V(2).Infof("Virtual machine %q is under deleting", key)
			return nil, nil
		}

		return &vm, nil
	}

	if az.VMCacheTTLInSeconds == 0 {
		az.VMCacheTTLInSeconds = vmCacheTTLDefaultInSeconds
	}
	return azcache.NewTimedcache(time.Duration(az.VMCacheTTLInSeconds)*time.Second, getter)
}


newLBCache

func (az *Cloud) newLBCache() (*azcache.TimedCache, error) {
	getter := func(key string) (interface{}, error) {
		ctx, cancel := getContextWithCancel()
		defer cancel()

		lb, err := az.LoadBalancerClient.Get(ctx, az.getLoadBalancerResourceGroup(), key, "")
		exists, rerr := checkResourceExistsFromError(err)
		if rerr != nil {
			return nil, rerr.Error()
		}

		if !exists {
			klog.V(2).Infof("Load balancer %q not found", key)
			return nil, nil
		}

		return &lb, nil
	}

	if az.LoadBalancerCacheTTLInSeconds == 0 {
		az.LoadBalancerCacheTTLInSeconds = loadBalancerCacheTTLDefaultInSeconds
	}
	return azcache.NewTimedcache(time.Duration(az.LoadBalancerCacheTTLInSeconds)*time.Second, getter)
}


newNSGCache

func (az *Cloud) newNSGCache() (*azcache.TimedCache, error) {
	getter := func(key string) (interface{}, error) {
		ctx, cancel := getContextWithCancel()
		defer cancel()
		nsg, err := az.SecurityGroupsClient.Get(ctx, az.SecurityGroupResourceGroup, key, "")
		exists, rerr := checkResourceExistsFromError(err)
		if rerr != nil {
			return nil, rerr.Error()
		}

		if !exists {
			klog.V(2).Infof("Security group %q not found", key)
			return nil, nil
		}

		return &nsg, nil
	}

	if az.NsgCacheTTLInSeconds == 0 {
		az.NsgCacheTTLInSeconds = nsgCacheTTLDefaultInSeconds
	}
	return azcache.NewTimedcache(time.Duration(az.NsgCacheTTLInSeconds)*time.Second, getter)
}


newRouteTableCache

func (az *Cloud) newRouteTableCache() (*azcache.TimedCache, error) {
	getter := func(key string) (interface{}, error) {
		ctx, cancel := getContextWithCancel()
		defer cancel()
		rt, err := az.RouteTablesClient.Get(ctx, az.RouteTableResourceGroup, key, "")
		exists, rerr := checkResourceExistsFromError(err)
		if rerr != nil {
			return nil, rerr.Error()
		}

		if !exists {
			klog.V(2).Infof("Route table %q not found", key)
			return nil, nil
		}

		return &rt, nil
	}

	if az.RouteTableCacheTTLInSeconds == 0 {
		az.RouteTableCacheTTLInSeconds = routeTableCacheTTLDefaultInSeconds
	}
	return azcache.NewTimedcache(time.Duration(az.RouteTableCacheTTLInSeconds)*time.Second, getter)
}


useStandardLoadBalancer

func (az *Cloud) useStandardLoadBalancer() bool {
	return strings.EqualFold(az.LoadBalancerSku, loadBalancerSkuStandard)
}


excludeMasterNodesFromStandardLB

func (az *Cloud) excludeMasterNodesFromStandardLB() bool {
	return az.ExcludeMasterFromStandardLB != nil && *az.ExcludeMasterFromStandardLB
}


disableLoadBalancerOutboundSNAT

func (az *Cloud) disableLoadBalancerOutboundSNAT() bool {
	if !az.useStandardLoadBalancer() || az.DisableOutboundSNAT == nil {
		return false
	}

	return *az.DisableOutboundSNAT
}


IsNodeUnmanaged

// IsNodeUnmanaged returns true if the node is not managed by Azure cloud provider.
// Those nodes includes on-prem or VMs from other clouds. They will not be added to load balancer
// backends. Azure routes and managed disks are also not supported for them.
func (az *Cloud) IsNodeUnmanaged(nodeName string) (bool, error) {
	unmanagedNodes, err := az.GetUnmanagedNodes()
	if err != nil {
		return false, err
	}

	return unmanagedNodes.Has(nodeName), nil
}


IsNodeUnmanagedByProviderID

// IsNodeUnmanagedByProviderID returns true if the node is not managed by Azure cloud provider.
// All managed node's providerIDs are in format 'azure:///subscriptions/<id>/resourceGroups/<rg>/providers/Microsoft.Compute/.*'
func (az *Cloud) IsNodeUnmanagedByProviderID(providerID string) bool {
	return !azureNodeProviderIDRE.Match([]byte(providerID))
}


convertResourceGroupNameToLower

// convertResourceGroupNameToLower converts the resource group name in the resource ID to be lowered.
func convertResourceGroupNameToLower(resourceID string) (string, error) {
	matches := azureResourceGroupNameRE.FindStringSubmatch(resourceID)
	if len(matches) != 2 {
		return "", fmt.Errorf("%q isn't in Azure resource ID format %q", resourceID, azureResourceGroupNameRE.String())
	}

	resourceGroup := matches[1]
	return strings.Replace(resourceID, resourceGroup, strings.ToLower(resourceGroup), 1), nil
}


isBackendPoolOnSameLB

// isBackendPoolOnSameLB checks whether newBackendPoolID is on the same load balancer as existingBackendPools.
// Since both public and internal LBs are supported, lbName and lbName-internal are treated as same.
// If not same, the lbName for existingBackendPools would also be returned.
func isBackendPoolOnSameLB(newBackendPoolID string, existingBackendPools []string) (bool, string, error) {
	matches := backendPoolIDRE.FindStringSubmatch(newBackendPoolID)
	if len(matches) != 2 {
		return false, "", fmt.Errorf("new backendPoolID %q is in wrong format", newBackendPoolID)
	}

	newLBName := matches[1]
	newLBNameTrimmed := strings.TrimSuffix(newLBName, InternalLoadBalancerNameSuffix)
	for _, backendPool := range existingBackendPools {
		matches := backendPoolIDRE.FindStringSubmatch(backendPool)
		if len(matches) != 2 {
			return false, "", fmt.Errorf("existing backendPoolID %q is in wrong format", backendPool)
		}

		lbName := matches[1]
		if !strings.EqualFold(strings.TrimSuffix(lbName, InternalLoadBalancerNameSuffix), newLBNameTrimmed) {
			return false, lbName, nil
		}
	}

	return true, "", nil
}


makeZone

// makeZone returns the zone value in format of <region>-<zone-id>.
func (az *Cloud) makeZone(location string, zoneID int) string {
	return fmt.Sprintf("%s-%d", strings.ToLower(location), zoneID)
}


isAvailabilityZone

// isAvailabilityZone returns true if the zone is in format of <region>-<zone-id>.
func (az *Cloud) isAvailabilityZone(zone string) bool {
	return strings.HasPrefix(zone, fmt.Sprintf("%s-", az.Location))
}


GetZoneID

// GetZoneID returns the ID of zone from node's zone label.
func (az *Cloud) GetZoneID(zoneLabel string) string {
	if !az.isAvailabilityZone(zoneLabel) {
		return ""
	}

	return strings.TrimPrefix(zoneLabel, fmt.Sprintf("%s-", az.Location))
}


GetZone

// GetZone returns the Zone containing the current availability zone and locality region that the program is running in.
// If the node is not running with availability zones, then it will fall back to fault domain.
func (az *Cloud) GetZone(ctx context.Context) (cloudprovider.Zone, error) {
	if az.UseInstanceMetadata {
		metadata, err := az.metadata.GetMetadata(azcache.CacheReadTypeUnsafe)
		if err != nil {
			return cloudprovider.Zone{}, err
		}

		if metadata.Compute == nil {
			az.metadata.imsCache.Delete(metadataCacheKey)
			return cloudprovider.Zone{}, fmt.Errorf("failure of getting compute information from instance metadata")
		}

		zone := ""
		location := metadata.Compute.Location
		if metadata.Compute.Zone != "" {
			zoneID, err := strconv.Atoi(metadata.Compute.Zone)
			if err != nil {
				return cloudprovider.Zone{}, fmt.Errorf("failed to parse zone ID %q: %v", metadata.Compute.Zone, err)
			}
			zone = az.makeZone(location, zoneID)
		} else {
			klog.V(3).Infof("Availability zone is not enabled for the node, falling back to fault domain")
			zone = metadata.Compute.FaultDomain
		}

		return cloudprovider.Zone{
			FailureDomain: strings.ToLower(zone),
			Region:        strings.ToLower(location),
		}, nil
	}
	// if UseInstanceMetadata is false, get Zone name by calling ARM
	hostname, err := os.Hostname()
	if err != nil {
		return cloudprovider.Zone{}, fmt.Errorf("failure getting hostname from kernel")
	}
	return az.VMSet.GetZoneByNodeName(strings.ToLower(hostname))
}


GetZoneByProviderID

// GetZoneByProviderID implements Zones.GetZoneByProviderID
// This is particularly useful in external cloud providers where the kubelet
// does not initialize node data.
func (az *Cloud) GetZoneByProviderID(ctx context.Context, providerID string) (cloudprovider.Zone, error) {
	if providerID == "" {
		return cloudprovider.Zone{}, errNodeNotInitialized
	}

	// Returns nil for unmanaged nodes because azure cloud provider couldn't fetch information for them.
	if az.IsNodeUnmanagedByProviderID(providerID) {
		klog.V(2).Infof("GetZoneByProviderID: omitting unmanaged node %q", providerID)
		return cloudprovider.Zone{}, nil
	}

	nodeName, err := az.VMSet.GetNodeNameByProviderID(providerID)
	if err != nil {
		return cloudprovider.Zone{}, err
	}

	return az.GetZoneByNodeName(ctx, nodeName)
}


GetZoneByNodeName

// GetZoneByNodeName implements Zones.GetZoneByNodeName
// This is particularly useful in external cloud providers where the kubelet
// does not initialize node data.
func (az *Cloud) GetZoneByNodeName(ctx context.Context, nodeName types.NodeName) (cloudprovider.Zone, error) {
	// Returns "" for unmanaged nodes because azure cloud provider couldn't fetch information for them.
	unmanaged, err := az.IsNodeUnmanaged(string(nodeName))
	if err != nil {
		return cloudprovider.Zone{}, err
	}
	if unmanaged {
		klog.V(2).Infof("GetZoneByNodeName: omitting unmanaged node %q", nodeName)
		return cloudprovider.Zone{}, nil
	}

	return az.VMSet.GetZoneByNodeName(string(nodeName))
}


cacheKeyFunc

// cacheKeyFunc defines the key function required in TTLStore.
func cacheKeyFunc(obj interface{}) (string, error) {
	return obj.(*AzureCacheEntry).Key, nil
}


NewTimedcache

// NewTimedcache creates a new TimedCache.
func NewTimedcache(ttl time.Duration, getter GetFunc) (*TimedCache, error) {
	if getter == nil {
		return nil, fmt.Errorf("getter is not provided")
	}

	return &TimedCache{
		Getter: getter,
		// switch to using NewStore instead of NewTTLStore so that we can
		// reuse entries for calls that are fine with reading expired/stalled data.
		// with NewTTLStore, entries are not returned if they have already expired.
		Store: cache.NewStore(cacheKeyFunc),
		TTL:   ttl,
	}, nil
}


getInternal

// getInternal returns AzureCacheEntry by key. If the key is not cached yet,
// it returns a AzureCacheEntry with nil data.
func (t *TimedCache) getInternal(key string) (*AzureCacheEntry, error) {
	entry, exists, err := t.Store.GetByKey(key)
	if err != nil {
		return nil, err
	}
	// if entry exists, return the entry
	if exists {
		return entry.(*AzureCacheEntry), nil
	}

	// lock here to ensure if entry doesn't exist, we add a new entry
	// avoiding overwrites
	t.Lock.Lock()
	defer t.Lock.Unlock()

	// Another goroutine might have written the same key.
	entry, exists, err = t.Store.GetByKey(key)
	if err != nil {
		return nil, err
	}
	if exists {
		return entry.(*AzureCacheEntry), nil
	}

	// Still not found, add new entry with nil data.
	// Note the data will be filled later by getter.
	newEntry := &AzureCacheEntry{
		Key:  key,
		Data: nil,
	}
	t.Store.Add(newEntry)
	return newEntry, nil
}


Get

// Get returns the requested item by key.
func (t *TimedCache) Get(key string, crt AzureCacheReadType) (interface{}, error) {
	entry, err := t.getInternal(key)
	if err != nil {
		return nil, err
	}

	entry.Lock.Lock()
	defer entry.Lock.Unlock()

	// entry exists and if cache is not force refreshed
	if entry.Data != nil && crt != CacheReadTypeForceRefresh {
		// allow unsafe read, so return data even if expired
		if crt == CacheReadTypeUnsafe {
			return entry.Data, nil
		}
		// if cached data is not expired, return cached data
		if crt == CacheReadTypeDefault && time.Since(entry.CreatedOn) < t.TTL {
			return entry.Data, nil
		}
	}
	// Data is not cached yet, cache data is expired or requested force refresh
	// cache it by getter. entry is locked before getting to ensure concurrent
	// gets don't result in multiple ARM calls.
	data, err := t.Getter(key)
	if err != nil {
		return nil, err
	}

	// set the data in cache and also set the last update time
	// to now as the data was recently fetched
	entry.Data = data
	entry.CreatedOn = time.Now().UTC()

	return entry.Data, nil
}


Delete

// Delete removes an item from the cache.
func (t *TimedCache) Delete(key string) error {
	return t.Store.Delete(&AzureCacheEntry{
		Key: key,
	})
}


Set

// Set sets the data cache for the key.
// It is only used for testing.
func (t *TimedCache) Set(key string, data interface{}) {
	t.Store.Add(&AzureCacheEntry{
		Key:       key,
		Data:      data,
		CreatedOn: time.Now().UTC(),
	})
}


New

// New creates a ARM client
func New(authorizer autorest.Authorizer, baseURI, userAgent, apiVersion, clientRegion string, clientBackoff *retry.Backoff) *Client {
	restClient := autorest.NewClientWithUserAgent(userAgent)
	restClient.PollingDelay = 5 * time.Second
	restClient.RetryAttempts = 3
	restClient.RetryDuration = time.Second * 1
	restClient.Authorizer = authorizer

	if userAgent == "" {
		restClient.UserAgent = GetUserAgent(restClient)
	}

	backoff := clientBackoff
	if backoff == nil {
		backoff = &retry.Backoff{}
	}
	if backoff.Steps == 0 {
		// 1 steps means no retry.
		backoff.Steps = 1
	}

	return &Client{
		client:       restClient,
		baseURI:      baseURI,
		backoff:      backoff,
		apiVersion:   apiVersion,
		clientRegion: NormalizeAzureRegion(clientRegion),
	}
}


GetUserAgent

// GetUserAgent gets the autorest client with a user agent that
// includes "kubernetes" and the full kubernetes git version string
// example:
// Azure-SDK-for-Go/7.0.1 arm-network/2016-09-01; kubernetes-cloudprovider/v1.17.0;
func GetUserAgent(client autorest.Client) string {
	k8sVersion := version.Get().GitVersion
	return fmt.Sprintf("%s; kubernetes-cloudprovider/%s", client.UserAgent, k8sVersion)
}


NormalizeAzureRegion

// NormalizeAzureRegion returns a normalized Azure region with white spaces removed and converted to lower case
func NormalizeAzureRegion(name string) string {
	region := ""
	for _, runeValue := range name {
		if !unicode.IsSpace(runeValue) {
			region += string(runeValue)
		}
	}
	return strings.ToLower(region)
}


sendRequest

// sendRequest sends a http request to ARM service.
// Although Azure SDK supports retries per https://github.com/azure/azure-sdk-for-go#request-retry-policy, we
// disable it since we want to fully control the retry policies.
func (c *Client) sendRequest(ctx context.Context, request *http.Request) (*http.Response, *retry.Error) {
	sendBackoff := *c.backoff
	response, err := autorest.SendWithSender(
		c.client,
		request,
		retry.DoExponentialBackoffRetry(&sendBackoff),
	)

	if response == nil && err == nil {
		return response, retry.NewError(false, fmt.Errorf("Empty response and no HTTP code"))
	}

	return response, retry.GetError(response, err)
}


Send

// Send sends a http request to ARM service with possible retry to regional ARM endpoint.
func (c *Client) Send(ctx context.Context, request *http.Request) (*http.Response, *retry.Error) {
	response, rerr := c.sendRequest(ctx, request)
	if rerr != nil {
		return response, rerr
	}

	if response.StatusCode != http.StatusNotFound || c.clientRegion == "" {
		return response, rerr
	}

	bodyBytes, _ := ioutil.ReadAll(response.Body)
	defer func() {
		response.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
	}()

	bodyString := string(bodyBytes)
	klog.V(5).Infof("Send.sendRequest original error message: %s", bodyString)

	// Hack: retry the regional ARM endpoint in case of ARM traffic split and arm resource group replication is too slow
	var body map[string]interface{}
	if e := json.Unmarshal(bodyBytes, &body); e != nil {
		klog.V(5).Infof("Send.sendRequest: error in parsing response body string: %s, Skip retrying regional host", e)
		return response, rerr
	}

	if err, ok := body["error"].(map[string]interface{}); !ok ||
		err["code"] == nil ||
		!strings.EqualFold(err["code"].(string), "ResourceGroupNotFound") {
		klog.V(5).Infof("Send.sendRequest: response body does not contain ResourceGroupNotFound error code. Skip retrying regional host")
		return response, rerr
	}

	currentHost := request.URL.Host
	if request.Host != "" {
		currentHost = request.Host
	}

	if strings.HasPrefix(strings.ToLower(currentHost), c.clientRegion) {
		klog.V(5).Infof("Send.sendRequest: current host %s is regional host. Skip retrying regional host.", currentHost)
		return response, rerr
	}

	request.Host = fmt.Sprintf("%s.%s", c.clientRegion, strings.ToLower(currentHost))
	klog.V(5).Infof("Send.sendRegionalRequest on ResourceGroupNotFound error. Retrying regional host: %s", request.Host)
	regionalResponse, regionalError := c.sendRequest(ctx, request)

	// only use the result if the regional request actually goes through and returns 2xx status code, for two reasons:
	// 1. the retry on regional ARM host approach is a hack.
	// 2. the concatenated regional uri could be wrong as the rule is not officially declared by ARM.
	if regionalResponse == nil || regionalResponse.StatusCode > 299 {
		regionalErrStr := ""
		if regionalError != nil {
			regionalErrStr = regionalError.Error().Error()
		}

		klog.V(5).Infof("Send.sendRegionalRequest failed to get response from regional host, error: '%s'. Ignoring the result.", regionalErrStr)
		return response, rerr
	}

	return regionalResponse, regionalError
}


PreparePutRequest

// PreparePutRequest prepares put request
func (c *Client) PreparePutRequest(ctx context.Context, decorators ...autorest.PrepareDecorator) (*http.Request, error) {
	decorators = append(
		[]autorest.PrepareDecorator{
			autorest.AsContentType("application/json; charset=utf-8"),
			autorest.AsPut(),
			autorest.WithBaseURL(c.baseURI)},
		decorators...)
	return c.prepareRequest(ctx, decorators...)
}


PreparePatchRequest

// PreparePatchRequest prepares patch request
func (c *Client) PreparePatchRequest(ctx context.Context, decorators ...autorest.PrepareDecorator) (*http.Request, error) {
	decorators = append(
		[]autorest.PrepareDecorator{
			autorest.AsContentType("application/json; charset=utf-8"),
			autorest.AsPatch(),
			autorest.WithBaseURL(c.baseURI)},
		decorators...)
	return c.prepareRequest(ctx, decorators...)
}


PreparePostRequest

// PreparePostRequest prepares post request
func (c *Client) PreparePostRequest(ctx context.Context, decorators ...autorest.PrepareDecorator) (*http.Request, error) {
	decorators = append(
		[]autorest.PrepareDecorator{
			autorest.AsContentType("application/json; charset=utf-8"),
			autorest.AsPost(),
			autorest.WithBaseURL(c.baseURI)},
		decorators...)
	return c.prepareRequest(ctx, decorators...)
}


PrepareGetRequest

// PrepareGetRequest prepares get request
func (c *Client) PrepareGetRequest(ctx context.Context, decorators ...autorest.PrepareDecorator) (*http.Request, error) {
	decorators = append(
		[]autorest.PrepareDecorator{
			autorest.AsGet(),
			autorest.WithBaseURL(c.baseURI)},
		decorators...)
	return c.prepareRequest(ctx, decorators...)
}


PrepareDeleteRequest

// PrepareDeleteRequest preparse delete request
func (c *Client) PrepareDeleteRequest(ctx context.Context, decorators ...autorest.PrepareDecorator) (*http.Request, error) {
	decorators = append(
		[]autorest.PrepareDecorator{
			autorest.AsDelete(),
			autorest.WithBaseURL(c.baseURI)},
		decorators...)
	return c.prepareRequest(ctx, decorators...)
}


PrepareHeadRequest

// PrepareHeadRequest prepares head request
func (c *Client) PrepareHeadRequest(ctx context.Context, decorators ...autorest.PrepareDecorator) (*http.Request, error) {
	decorators = append(
		[]autorest.PrepareDecorator{
			autorest.AsHead(),
			autorest.WithBaseURL(c.baseURI)},
		decorators...)
	return c.prepareRequest(ctx, decorators...)
}


WaitForAsyncOperationCompletion

// WaitForAsyncOperationCompletion waits for an operation completion
func (c *Client) WaitForAsyncOperationCompletion(ctx context.Context, future *azure.Future, asyncOperationName string) error {
	err := future.WaitForCompletionRef(ctx, c.client)
	if err != nil {
		klog.V(5).Infof("Received error in WaitForCompletionRef: '%v'", err)
		return err
	}

	var done bool
	done, err = future.DoneWithContext(ctx, c.client)
	if err != nil {
		klog.V(5).Infof("Received error in DoneWithContext: '%v'", err)
		return autorest.NewErrorWithError(err, asyncOperationName, "Result", future.Response(), "Polling failure")
	}
	if !done {
		return azure.NewAsyncOpIncompleteError(asyncOperationName)
	}

	return nil
}


WaitForAsyncOperationResult

// WaitForAsyncOperationResult waits for an operation result.
func (c *Client) WaitForAsyncOperationResult(ctx context.Context, future *azure.Future, asyncOperationName string) (*http.Response, error) {
	err := c.WaitForAsyncOperationCompletion(ctx, future, asyncOperationName)
	if err != nil {
		klog.V(5).Infof("Received error in WaitForAsyncOperationCompletion: '%v'", err)
		return nil, err
	}

	sendBackoff := *c.backoff
	sender := autorest.DecorateSender(
		c.client,
		retry.DoExponentialBackoffRetry(&sendBackoff),
	)
	return future.GetResult(sender)
}


SendAsync

// SendAsync send a request and return a future object representing the async result as well as the origin http response
func (c *Client) SendAsync(ctx context.Context, request *http.Request) (*azure.Future, *http.Response, *retry.Error) {
	asyncResponse, rerr := c.Send(ctx, request)
	if rerr != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "sendAsync.send", request.URL.String(), rerr.Error())
		return nil, nil, rerr
	}

	future, err := azure.NewFutureFromResponse(asyncResponse)
	if err != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "sendAsync.respond", request.URL.String(), err)
		return nil, asyncResponse, retry.GetError(asyncResponse, err)
	}

	return &future, asyncResponse, nil
}


GetResource

// GetResource get a resource by resource ID
func (c *Client) GetResource(ctx context.Context, resourceID, expand string) (*http.Response, *retry.Error) {
	decorators := []autorest.PrepareDecorator{
		autorest.WithPathParameters("{resourceID}", map[string]interface{}{"resourceID": resourceID}),
	}
	if expand != "" {
		queryParameters := map[string]interface{}{
			"$expand": autorest.Encode("query", expand),
		}
		decorators = append(decorators, autorest.WithQueryParameters(queryParameters))
	}
	request, err := c.PrepareGetRequest(ctx, decorators...)
	if err != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "get.prepare", resourceID, err)
		return nil, retry.NewError(false, err)
	}

	return c.Send(ctx, request)
}


GetResourceWithDecorators

// GetResourceWithDecorators get a resource with decorators by resource ID
func (c *Client) GetResourceWithDecorators(ctx context.Context, resourceID string, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error) {
	getDecorators := []autorest.PrepareDecorator{
		autorest.WithPathParameters("{resourceID}", map[string]interface{}{"resourceID": resourceID}),
	}
	getDecorators = append(getDecorators, decorators...)
	request, err := c.PrepareGetRequest(ctx, getDecorators...)
	if err != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "get.prepare", resourceID, err)
		return nil, retry.NewError(false, err)
	}

	return c.Send(ctx, request)
}


PutResource

// PutResource puts a resource by resource ID
func (c *Client) PutResource(ctx context.Context, resourceID string, parameters interface{}) (*http.Response, *retry.Error) {
	putDecorators := []autorest.PrepareDecorator{
		autorest.WithPathParameters("{resourceID}", map[string]interface{}{"resourceID": resourceID}),
		autorest.WithJSON(parameters),
	}
	return c.PutResourceWithDecorators(ctx, resourceID, parameters, putDecorators)
}


PutResources

// PutResources puts a list of resources from resources map[resourceID]parameters.
// Those resources sync requests are sequential while async requests are concurrent. It's especially
// useful when the ARM API doesn't support concurrent requests.
func (c *Client) PutResources(ctx context.Context, resources map[string]interface{}) map[string]*PutResourcesResponse {
	if len(resources) == 0 {
		return nil
	}

	// Sequential sync requests.
	futures := make(map[string]*azure.Future)
	responses := make(map[string]*PutResourcesResponse)
	for resourceID, parameters := range resources {
		decorators := []autorest.PrepareDecorator{
			autorest.WithPathParameters("{resourceID}", map[string]interface{}{"resourceID": resourceID}),
			autorest.WithJSON(parameters),
		}
		request, err := c.PreparePutRequest(ctx, decorators...)
		if err != nil {
			klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "put.prepare", resourceID, err)
			responses[resourceID] = &PutResourcesResponse{
				Error: retry.NewError(false, err),
			}
			continue
		}

		future, resp, clientErr := c.SendAsync(ctx, request)
		defer c.CloseResponse(ctx, resp)
		if clientErr != nil {
			klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "put.send", resourceID, clientErr.Error())
			responses[resourceID] = &PutResourcesResponse{
				Error: clientErr,
			}
			continue
		}

		futures[resourceID] = future
	}

	// Concurrent async requests.
	wg := sync.WaitGroup{}
	var responseLock sync.Mutex
	for resourceID, future := range futures {
		wg.Add(1)
		go func(resourceID string, future *azure.Future) {
			defer wg.Done()
			response, err := c.WaitForAsyncOperationResult(ctx, future, "armclient.PutResource")
			if err != nil {
				if response != nil {
					klog.V(5).Infof("Received error in WaitForAsyncOperationResult: '%s', response code %d", err.Error(), response.StatusCode)
				} else {
					klog.V(5).Infof("Received error in WaitForAsyncOperationResult: '%s', no response", err.Error())
				}

				retriableErr := retry.GetError(response, err)
				if !retriableErr.Retriable &&
					strings.Contains(strings.ToUpper(err.Error()), strings.ToUpper("InternalServerError")) {
					klog.V(5).Infof("Received InternalServerError in WaitForAsyncOperationResult: '%s', setting error retriable", err.Error())
					retriableErr.Retriable = true
				}

				responseLock.Lock()
				responses[resourceID] = &PutResourcesResponse{
					Error: retriableErr,
				}
				responseLock.Unlock()
				return
			}

			responseLock.Lock()
			responses[resourceID] = &PutResourcesResponse{
				Response: response,
			}
			responseLock.Unlock()
		}(resourceID, future)
	}

	wg.Wait()
	return responses
}


PutResourceWithDecorators

// PutResourceWithDecorators puts a resource by resource ID
func (c *Client) PutResourceWithDecorators(ctx context.Context, resourceID string, parameters interface{}, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error) {
	request, err := c.PreparePutRequest(ctx, decorators...)
	if err != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "put.prepare", resourceID, err)
		return nil, retry.NewError(false, err)
	}

	future, resp, clientErr := c.SendAsync(ctx, request)
	defer c.CloseResponse(ctx, resp)
	if clientErr != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "put.send", resourceID, clientErr.Error())
		return nil, clientErr
	}

	response, err := c.WaitForAsyncOperationResult(ctx, future, "armclient.PutResource")
	if err != nil {
		if response != nil {
			klog.V(5).Infof("Received error in WaitForAsyncOperationResult: '%s', response code %d", err.Error(), response.StatusCode)
		} else {
			klog.V(5).Infof("Received error in WaitForAsyncOperationResult: '%s', no response", err.Error())
		}

		retriableErr := retry.GetError(response, err)
		if !retriableErr.Retriable &&
			strings.Contains(strings.ToUpper(err.Error()), strings.ToUpper("InternalServerError")) {
			klog.V(5).Infof("Received InternalServerError in WaitForAsyncOperationResult: '%s', setting error retriable", err.Error())
			retriableErr.Retriable = true
		}
		return nil, retriableErr
	}

	return response, nil
}


PatchResource

// PatchResource patches a resource by resource ID
func (c *Client) PatchResource(ctx context.Context, resourceID string, parameters interface{}) (*http.Response, *retry.Error) {
	decorators := []autorest.PrepareDecorator{
		autorest.WithPathParameters("{resourceID}", map[string]interface{}{"resourceID": resourceID}),
		autorest.WithJSON(parameters),
	}

	request, err := c.PreparePatchRequest(ctx, decorators...)
	if err != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "patch.prepare", resourceID, err)
		return nil, retry.NewError(false, err)
	}

	future, resp, clientErr := c.SendAsync(ctx, request)
	defer c.CloseResponse(ctx, resp)
	if clientErr != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "patch.send", resourceID, clientErr.Error())
		return nil, clientErr
	}

	response, err := c.WaitForAsyncOperationResult(ctx, future, "armclient.PatchResource")
	if err != nil {
		if response != nil {
			klog.V(5).Infof("Received error in WaitForAsyncOperationResult: '%s', response code %d", err.Error(), response.StatusCode)
		} else {
			klog.V(5).Infof("Received error in WaitForAsyncOperationResult: '%s', no response", err.Error())
		}

		retriableErr := retry.GetError(response, err)
		if !retriableErr.Retriable &&
			strings.Contains(strings.ToUpper(err.Error()), strings.ToUpper("InternalServerError")) {
			klog.V(5).Infof("Received InternalServerError in WaitForAsyncOperationResult: '%s', setting error retriable", err.Error())
			retriableErr.Retriable = true
		}
		return nil, retriableErr
	}

	return response, nil
}


PutResourceAsync

// PutResourceAsync puts a resource by resource ID in async mode
func (c *Client) PutResourceAsync(ctx context.Context, resourceID string, parameters interface{}) (*azure.Future, *retry.Error) {
	decorators := []autorest.PrepareDecorator{
		autorest.WithPathParameters("{resourceID}", map[string]interface{}{"resourceID": resourceID}),
		autorest.WithJSON(parameters),
	}

	request, err := c.PreparePutRequest(ctx, decorators...)
	if err != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "put.prepare", resourceID, err)
		return nil, retry.NewError(false, err)
	}

	future, resp, rErr := c.SendAsync(ctx, request)
	defer c.CloseResponse(ctx, resp)
	if rErr != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "put.send", resourceID, err)
		return nil, rErr
	}

	return future, nil
}


PostResource

// PostResource posts a resource by resource ID
func (c *Client) PostResource(ctx context.Context, resourceID, action string, parameters interface{}) (*http.Response, *retry.Error) {
	pathParameters := map[string]interface{}{
		"resourceID": resourceID,
		"action":     action,
	}

	decorators := []autorest.PrepareDecorator{
		autorest.WithPathParameters("{resourceID}/{action}", pathParameters),
		autorest.WithJSON(parameters),
	}
	request, err := c.PreparePostRequest(ctx, decorators...)
	if err != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "post.prepare", resourceID, err)
		return nil, retry.NewError(false, err)
	}

	return c.sendRequest(ctx, request)
}


DeleteResource

// DeleteResource deletes a resource by resource ID
func (c *Client) DeleteResource(ctx context.Context, resourceID, ifMatch string) *retry.Error {
	future, clientErr := c.DeleteResourceAsync(ctx, resourceID, ifMatch)
	if clientErr != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "delete.request", resourceID, clientErr.Error())
		return clientErr
	}

	if future == nil {
		return nil
	}

	if err := c.WaitForAsyncOperationCompletion(ctx, future, "armclient.DeleteResource"); err != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "delete.wait", resourceID, clientErr.Error())
		return retry.NewError(true, err)
	}

	return nil
}


HeadResource

// HeadResource heads a resource by resource ID
func (c *Client) HeadResource(ctx context.Context, resourceID string) (*http.Response, *retry.Error) {
	decorators := []autorest.PrepareDecorator{
		autorest.WithPathParameters("{resourceID}", map[string]interface{}{"resourceID": resourceID}),
	}
	request, err := c.PrepareHeadRequest(ctx, decorators...)
	if err != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "head.prepare", resourceID, err)
		return nil, retry.NewError(false, err)
	}

	return c.sendRequest(ctx, request)
}


DeleteResourceAsync

// DeleteResourceAsync delete a resource by resource ID and returns a future representing the async result
func (c *Client) DeleteResourceAsync(ctx context.Context, resourceID, ifMatch string) (*azure.Future, *retry.Error) {
	decorators := []autorest.PrepareDecorator{
		autorest.WithPathParameters("{resourceID}", map[string]interface{}{"resourceID": resourceID}),
	}
	if len(ifMatch) > 0 {
		decorators = append(decorators, autorest.WithHeader("If-Match", autorest.String(ifMatch)))
	}

	deleteRequest, err := c.PrepareDeleteRequest(ctx, decorators...)
	if err != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "deleteAsync.prepare", resourceID, err)
		return nil, retry.NewError(false, err)
	}

	resp, rerr := c.sendRequest(ctx, deleteRequest)
	defer c.CloseResponse(ctx, resp)
	if rerr != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "deleteAsync.send", resourceID, rerr.Error())
		return nil, rerr
	}

	err = autorest.Respond(
		resp,
		azure.WithErrorUnlessStatusCode(http.StatusOK, http.StatusAccepted, http.StatusNoContent, http.StatusNotFound))
	if err != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "deleteAsync.respond", resourceID, err)
		return nil, retry.GetError(resp, err)
	}

	if resp.StatusCode == http.StatusNotFound {
		return nil, nil
	}

	future, err := azure.NewFutureFromResponse(resp)
	if err != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "deleteAsync.future", resourceID, err)
		return nil, retry.GetError(resp, err)
	}

	return &future, nil
}


CloseResponse

// CloseResponse closes a response
func (c *Client) CloseResponse(ctx context.Context, response *http.Response) {
	if response != nil && response.Body != nil {
		if err := response.Body.Close(); err != nil {
			klog.Errorf("Error closing the response body: %v", err)
		}
	}
}


prepareRequest

func (c *Client) prepareRequest(ctx context.Context, decorators ...autorest.PrepareDecorator) (*http.Request, error) {
	decorators = append(
		decorators,
		withAPIVersion(c.apiVersion))
	preparer := autorest.CreatePreparer(decorators...)
	return preparer.Prepare((&http.Request{}).WithContext(ctx))
}


withAPIVersion

func withAPIVersion(apiVersion string) autorest.PrepareDecorator {
	const apiVersionKey = "api-version"
	return func(p autorest.Preparer) autorest.Preparer {
		return autorest.PreparerFunc(func(r *http.Request) (*http.Request, error) {
			r, err := p.Prepare(r)
			if err == nil {
				if r.URL == nil {
					return r, fmt.Errorf("Error in withAPIVersion: Invoked with a nil URL")
				}

				v := r.URL.Query()
				if len(v.Get(apiVersionKey)) > 0 {
					return r, nil
				}

				v.Add(apiVersionKey, apiVersion)
				r.URL.RawQuery = v.Encode()
			}
			return r, err
		})
	}
}


GetResourceID

// GetResourceID gets Azure resource ID
func GetResourceID(subscriptionID, resourceGroupName, resourceType, resourceName string) string {
	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/%s/%s",
		autorest.Encode("path", subscriptionID),
		autorest.Encode("path", resourceGroupName),
		resourceType,
		autorest.Encode("path", resourceName))
}


GetChildResourceID

// GetChildResourceID gets Azure child resource ID
func GetChildResourceID(subscriptionID, resourceGroupName, resourceType, resourceName, childResourceType, childResourceName string) string {
	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/%s/%s/%s/%s",
		autorest.Encode("path", subscriptionID),
		autorest.Encode("path", resourceGroupName),
		resourceType,
		autorest.Encode("path", resourceName),
		childResourceType,
		autorest.Encode("path", childResourceName))
}


GetChildResourcesListID

// GetChildResourcesListID gets Azure child resources list ID
func GetChildResourcesListID(subscriptionID, resourceGroupName, resourceType, resourceName, childResourceType string) string {
	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/%s/%s/%s",
		autorest.Encode("path", subscriptionID),
		autorest.Encode("path", resourceGroupName),
		resourceType,
		autorest.Encode("path", resourceName),
		childResourceType)
}


GetProviderResourceID

// GetProviderResourceID gets Azure RP resource ID
func GetProviderResourceID(subscriptionID, providerNamespace string) string {
	return fmt.Sprintf("/subscriptions/%s/providers/%s",
		autorest.Encode("path", subscriptionID),
		providerNamespace)
}


GetProviderResourcesListID

// GetProviderResourcesListID gets Azure RP resources list ID
func GetProviderResourcesListID(subscriptionID string) string {
	return fmt.Sprintf("/subscriptions/%s/providers", autorest.Encode("path", subscriptionID))
}


NewMockInterface

// NewMockInterface creates a new mock instance.
func NewMockInterface(ctrl *gomock.Controller) *MockInterface {
	mock := &MockInterface{ctrl: ctrl}
	mock.recorder = &MockInterfaceMockRecorder{mock}
	return mock
}


EXPECT

// EXPECT returns an object that allows the caller to indicate expected use.
func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder {
	return m.recorder
}


CloseResponse

// CloseResponse mocks base method.
func (m *MockInterface) CloseResponse(ctx context.Context, response *http.Response) {
	m.ctrl.T.Helper()
	m.ctrl.Call(m, "CloseResponse", ctx, response)
}


// CloseResponse indicates an expected call of CloseResponse.
func (mr *MockInterfaceMockRecorder) CloseResponse(ctx, response interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloseResponse", reflect.TypeOf((*MockInterface)(nil).CloseResponse), ctx, response)
}


DeleteResource

// DeleteResource mocks base method.
func (m *MockInterface) DeleteResource(ctx context.Context, resourceID, ifMatch string) *retry.Error {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "DeleteResource", ctx, resourceID, ifMatch)
	ret0, _ := ret[0].(*retry.Error)
	return ret0
}


// DeleteResource indicates an expected call of DeleteResource.
func (mr *MockInterfaceMockRecorder) DeleteResource(ctx, resourceID, ifMatch interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteResource", reflect.TypeOf((*MockInterface)(nil).DeleteResource), ctx, resourceID, ifMatch)
}


DeleteResourceAsync

// DeleteResourceAsync mocks base method.
func (m *MockInterface) DeleteResourceAsync(ctx context.Context, resourceID, ifMatch string) (*azure.Future, *retry.Error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "DeleteResourceAsync", ctx, resourceID, ifMatch)
	ret0, _ := ret[0].(*azure.Future)
	ret1, _ := ret[1].(*retry.Error)
	return ret0, ret1
}


// DeleteResourceAsync indicates an expected call of DeleteResourceAsync.
func (mr *MockInterfaceMockRecorder) DeleteResourceAsync(ctx, resourceID, ifMatch interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteResourceAsync", reflect.TypeOf((*MockInterface)(nil).DeleteResourceAsync), ctx, resourceID, ifMatch)
}


GetResource

// GetResource mocks base method.
func (m *MockInterface) GetResource(ctx context.Context, resourceID, expand string) (*http.Response, *retry.Error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "GetResource", ctx, resourceID, expand)
	ret0, _ := ret[0].(*http.Response)
	ret1, _ := ret[1].(*retry.Error)
	return ret0, ret1
}


// GetResource indicates an expected call of GetResource.
func (mr *MockInterfaceMockRecorder) GetResource(ctx, resourceID, expand interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResource", reflect.TypeOf((*MockInterface)(nil).GetResource), ctx, resourceID, expand)
}


GetResourceWithDecorators

// GetResourceWithDecorators mocks base method.
func (m *MockInterface) GetResourceWithDecorators(ctx context.Context, resourceID string, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "GetResourceWithDecorators", ctx, resourceID, decorators)
	ret0, _ := ret[0].(*http.Response)
	ret1, _ := ret[1].(*retry.Error)
	return ret0, ret1
}


// GetResourceWithDecorators indicates an expected call of GetResourceWithDecorators.
func (mr *MockInterfaceMockRecorder) GetResourceWithDecorators(ctx, resourceID, decorators interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResourceWithDecorators", reflect.TypeOf((*MockInterface)(nil).GetResourceWithDecorators), ctx, resourceID, decorators)
}


HeadResource

// HeadResource mocks base method.
func (m *MockInterface) HeadResource(ctx context.Context, resourceID string) (*http.Response, *retry.Error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "HeadResource", ctx, resourceID)
	ret0, _ := ret[0].(*http.Response)
	ret1, _ := ret[1].(*retry.Error)
	return ret0, ret1
}


// HeadResource indicates an expected call of HeadResource.
func (mr *MockInterfaceMockRecorder) HeadResource(ctx, resourceID interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HeadResource", reflect.TypeOf((*MockInterface)(nil).HeadResource), ctx, resourceID)
}


PatchResource

// PatchResource mocks base method.
func (m *MockInterface) PatchResource(ctx context.Context, resourceID string, parameters interface{}) (*http.Response, *retry.Error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "PatchResource", ctx, resourceID, parameters)
	ret0, _ := ret[0].(*http.Response)
	ret1, _ := ret[1].(*retry.Error)
	return ret0, ret1
}


// PatchResource indicates an expected call of PatchResource.
func (mr *MockInterfaceMockRecorder) PatchResource(ctx, resourceID, parameters interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PatchResource", reflect.TypeOf((*MockInterface)(nil).PatchResource), ctx, resourceID, parameters)
}


PostResource

// PostResource mocks base method.
func (m *MockInterface) PostResource(ctx context.Context, resourceID, action string, parameters interface{}) (*http.Response, *retry.Error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "PostResource", ctx, resourceID, action, parameters)
	ret0, _ := ret[0].(*http.Response)
	ret1, _ := ret[1].(*retry.Error)
	return ret0, ret1
}


// PostResource indicates an expected call of PostResource.
func (mr *MockInterfaceMockRecorder) PostResource(ctx, resourceID, action, parameters interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PostResource", reflect.TypeOf((*MockInterface)(nil).PostResource), ctx, resourceID, action, parameters)
}


PrepareDeleteRequest

// PrepareDeleteRequest mocks base method.
func (m *MockInterface) PrepareDeleteRequest(ctx context.Context, decorators ...autorest.PrepareDecorator) (*http.Request, error) {
	m.ctrl.T.Helper()
	varargs := []interface{}{ctx}
	for _, a := range decorators {
		varargs = append(varargs, a)
	}
	ret := m.ctrl.Call(m, "PrepareDeleteRequest", varargs...)
	ret0, _ := ret[0].(*http.Request)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}


// PrepareDeleteRequest indicates an expected call of PrepareDeleteRequest.
func (mr *MockInterfaceMockRecorder) PrepareDeleteRequest(ctx interface{}, decorators ...interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	varargs := append([]interface{}{ctx}, decorators...)
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PrepareDeleteRequest", reflect.TypeOf((*MockInterface)(nil).PrepareDeleteRequest), varargs...)
}


PrepareGetRequest

// PrepareGetRequest mocks base method.
func (m *MockInterface) PrepareGetRequest(ctx context.Context, decorators ...autorest.PrepareDecorator) (*http.Request, error) {
	m.ctrl.T.Helper()
	varargs := []interface{}{ctx}
	for _, a := range decorators {
		varargs = append(varargs, a)
	}
	ret := m.ctrl.Call(m, "PrepareGetRequest", varargs...)
	ret0, _ := ret[0].(*http.Request)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}


// PrepareGetRequest indicates an expected call of PrepareGetRequest.
func (mr *MockInterfaceMockRecorder) PrepareGetRequest(ctx interface{}, decorators ...interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	varargs := append([]interface{}{ctx}, decorators...)
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PrepareGetRequest", reflect.TypeOf((*MockInterface)(nil).PrepareGetRequest), varargs...)
}


PrepareHeadRequest

// PrepareHeadRequest mocks base method.
func (m *MockInterface) PrepareHeadRequest(ctx context.Context, decorators ...autorest.PrepareDecorator) (*http.Request, error) {
	m.ctrl.T.Helper()
	varargs := []interface{}{ctx}
	for _, a := range decorators {
		varargs = append(varargs, a)
	}
	ret := m.ctrl.Call(m, "PrepareHeadRequest", varargs...)
	ret0, _ := ret[0].(*http.Request)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}


// PrepareHeadRequest indicates an expected call of PrepareHeadRequest.
func (mr *MockInterfaceMockRecorder) PrepareHeadRequest(ctx interface{}, decorators ...interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	varargs := append([]interface{}{ctx}, decorators...)
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PrepareHeadRequest", reflect.TypeOf((*MockInterface)(nil).PrepareHeadRequest), varargs...)
}


PreparePostRequest

// PreparePostRequest mocks base method.
func (m *MockInterface) PreparePostRequest(ctx context.Context, decorators ...autorest.PrepareDecorator) (*http.Request, error) {
	m.ctrl.T.Helper()
	varargs := []interface{}{ctx}
	for _, a := range decorators {
		varargs = append(varargs, a)
	}
	ret := m.ctrl.Call(m, "PreparePostRequest", varargs...)
	ret0, _ := ret[0].(*http.Request)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}


// PreparePostRequest indicates an expected call of PreparePostRequest.
func (mr *MockInterfaceMockRecorder) PreparePostRequest(ctx interface{}, decorators ...interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	varargs := append([]interface{}{ctx}, decorators...)
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PreparePostRequest", reflect.TypeOf((*MockInterface)(nil).PreparePostRequest), varargs...)
}


PreparePutRequest

// PreparePutRequest mocks base method.
func (m *MockInterface) PreparePutRequest(ctx context.Context, decorators ...autorest.PrepareDecorator) (*http.Request, error) {
	m.ctrl.T.Helper()
	varargs := []interface{}{ctx}
	for _, a := range decorators {
		varargs = append(varargs, a)
	}
	ret := m.ctrl.Call(m, "PreparePutRequest", varargs...)
	ret0, _ := ret[0].(*http.Request)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}


// PreparePutRequest indicates an expected call of PreparePutRequest.
func (mr *MockInterfaceMockRecorder) PreparePutRequest(ctx interface{}, decorators ...interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	varargs := append([]interface{}{ctx}, decorators...)
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PreparePutRequest", reflect.TypeOf((*MockInterface)(nil).PreparePutRequest), varargs...)
}


PutResource

// PutResource mocks base method.
func (m *MockInterface) PutResource(ctx context.Context, resourceID string, parameters interface{}) (*http.Response, *retry.Error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "PutResource", ctx, resourceID, parameters)
	ret0, _ := ret[0].(*http.Response)
	ret1, _ := ret[1].(*retry.Error)
	return ret0, ret1
}


// PutResource indicates an expected call of PutResource.
func (mr *MockInterfaceMockRecorder) PutResource(ctx, resourceID, parameters interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutResource", reflect.TypeOf((*MockInterface)(nil).PutResource), ctx, resourceID, parameters)
}


PutResourceAsync

// PutResourceAsync mocks base method.
func (m *MockInterface) PutResourceAsync(ctx context.Context, resourceID string, parameters interface{}) (*azure.Future, *retry.Error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "PutResourceAsync", ctx, resourceID, parameters)
	ret0, _ := ret[0].(*azure.Future)
	ret1, _ := ret[1].(*retry.Error)
	return ret0, ret1
}


// PutResourceAsync indicates an expected call of PutResourceAsync.
func (mr *MockInterfaceMockRecorder) PutResourceAsync(ctx, resourceID, parameters interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutResourceAsync", reflect.TypeOf((*MockInterface)(nil).PutResourceAsync), ctx, resourceID, parameters)
}


PutResourceWithDecorators

// PutResourceWithDecorators mocks base method.
func (m *MockInterface) PutResourceWithDecorators(ctx context.Context, resourceID string, parameters interface{}, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "PutResourceWithDecorators", ctx, resourceID, parameters, decorators)
	ret0, _ := ret[0].(*http.Response)
	ret1, _ := ret[1].(*retry.Error)
	return ret0, ret1
}


// PutResourceWithDecorators indicates an expected call of PutResourceWithDecorators.
func (mr *MockInterfaceMockRecorder) PutResourceWithDecorators(ctx, resourceID, parameters, decorators interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutResourceWithDecorators", reflect.TypeOf((*MockInterface)(nil).PutResourceWithDecorators), ctx, resourceID, parameters, decorators)
}


PutResources

// PutResources mocks base method.
func (m *MockInterface) PutResources(ctx context.Context, resources map[string]interface{}) map[string]*armclient.PutResourcesResponse {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "PutResources", ctx, resources)
	ret0, _ := ret[0].(map[string]*armclient.PutResourcesResponse)
	return ret0
}


// PutResources indicates an expected call of PutResources.
func (mr *MockInterfaceMockRecorder) PutResources(ctx, resources interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutResources", reflect.TypeOf((*MockInterface)(nil).PutResources), ctx, resources)
}


Send

// Send mocks base method.
func (m *MockInterface) Send(ctx context.Context, request *http.Request) (*http.Response, *retry.Error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "Send", ctx, request)
	ret0, _ := ret[0].(*http.Response)
	ret1, _ := ret[1].(*retry.Error)
	return ret0, ret1
}


// Send indicates an expected call of Send.
func (mr *MockInterfaceMockRecorder) Send(ctx, request interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockInterface)(nil).Send), ctx, request)
}


SendAsync

// SendAsync mocks base method.
func (m *MockInterface) SendAsync(ctx context.Context, request *http.Request) (*azure.Future, *http.Response, *retry.Error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "SendAsync", ctx, request)
	ret0, _ := ret[0].(*azure.Future)
	ret1, _ := ret[1].(*http.Response)
	ret2, _ := ret[2].(*retry.Error)
	return ret0, ret1, ret2
}


// SendAsync indicates an expected call of SendAsync.
func (mr *MockInterfaceMockRecorder) SendAsync(ctx, request interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendAsync", reflect.TypeOf((*MockInterface)(nil).SendAsync), ctx, request)
}


WaitForAsyncOperationCompletion

// WaitForAsyncOperationCompletion mocks base method.
func (m *MockInterface) WaitForAsyncOperationCompletion(ctx context.Context, future *azure.Future, asyncOperationName string) error {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "WaitForAsyncOperationCompletion", ctx, future, asyncOperationName)
	ret0, _ := ret[0].(error)
	return ret0
}


// WaitForAsyncOperationCompletion indicates an expected call of WaitForAsyncOperationCompletion.
func (mr *MockInterfaceMockRecorder) WaitForAsyncOperationCompletion(ctx, future, asyncOperationName interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForAsyncOperationCompletion", reflect.TypeOf((*MockInterface)(nil).WaitForAsyncOperationCompletion), ctx, future, asyncOperationName)
}


WaitForAsyncOperationResult

// WaitForAsyncOperationResult mocks base method.
func (m *MockInterface) WaitForAsyncOperationResult(ctx context.Context, future *azure.Future, asyncOperationName string) (*http.Response, error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "WaitForAsyncOperationResult", ctx, future, asyncOperationName)
	ret0, _ := ret[0].(*http.Response)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}


// WaitForAsyncOperationResult indicates an expected call of WaitForAsyncOperationResult.
func (mr *MockInterfaceMockRecorder) WaitForAsyncOperationResult(ctx, future, asyncOperationName interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForAsyncOperationResult", reflect.TypeOf((*MockInterface)(nil).WaitForAsyncOperationResult), ctx, future, asyncOperationName)
}


WithRateLimiter

// WithRateLimiter returns a new ClientConfig with rateLimitConfig set.
func (cfg *ClientConfig) WithRateLimiter(rl *RateLimitConfig) *ClientConfig {
	newClientConfig := *cfg
	newClientConfig.RateLimitConfig = rl
	return &newClientConfig
}


RateLimitEnabled

// RateLimitEnabled returns true if CloudProviderRateLimit is set to true.
func RateLimitEnabled(config *RateLimitConfig) bool {
	return config != nil && config.CloudProviderRateLimit
}


NewRateLimiter

// NewRateLimiter creates new read and write flowcontrol.RateLimiter from RateLimitConfig.
func NewRateLimiter(config *RateLimitConfig) (flowcontrol.RateLimiter, flowcontrol.RateLimiter) {
	readLimiter := flowcontrol.NewFakeAlwaysRateLimiter()
	writeLimiter := flowcontrol.NewFakeAlwaysRateLimiter()

	if config != nil && config.CloudProviderRateLimit {
		readLimiter = flowcontrol.NewTokenBucketRateLimiter(
			config.CloudProviderRateLimitQPS,
			config.CloudProviderRateLimitBucket)

		writeLimiter = flowcontrol.NewTokenBucketRateLimiter(
			config.CloudProviderRateLimitQPSWrite,
			config.CloudProviderRateLimitBucketWrite)
	}

	return readLimiter, writeLimiter
}


New

// New creates a new ContainerServiceClient client with ratelimiting.
func New(config *azclients.ClientConfig) *Client {
	baseURI := config.ResourceManagerEndpoint
	authorizer := config.Authorizer
	armClient := armclient.New(authorizer, baseURI, config.UserAgent, APIVersion, config.Location, config.Backoff)
	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(config.RateLimitConfig)

	klog.V(2).Infof("Azure ContainerServiceClient (read ops) using rate limit config: QPS=%g, bucket=%d",
		config.RateLimitConfig.CloudProviderRateLimitQPS,
		config.RateLimitConfig.CloudProviderRateLimitBucket)
	klog.V(2).Infof("Azure ContainerServiceClient (write ops) using rate limit config: QPS=%g, bucket=%d",
		config.RateLimitConfig.CloudProviderRateLimitQPSWrite,
		config.RateLimitConfig.CloudProviderRateLimitBucketWrite)

	client := &Client{
		armClient:         armClient,
		rateLimiterReader: rateLimiterReader,
		rateLimiterWriter: rateLimiterWriter,
		subscriptionID:    config.SubscriptionID,
	}

	return client
}


Get

// Get gets a ManagedCluster.
func (c *Client) Get(ctx context.Context, resourceGroupName string, managedClusterName string) (containerservice.ManagedCluster, *retry.Error) {
	mc := metrics.NewMetricContext("managed_clusters", "get", resourceGroupName, c.subscriptionID, "")

	// Report errors if the client is rate limited.
	if !c.rateLimiterReader.TryAccept() {
		mc.RateLimitedCount()
		return containerservice.ManagedCluster{}, retry.GetRateLimitError(false, "GetManagedCluster")
	}

	// Report errors if the client is throttled.
	if c.RetryAfterReader.After(time.Now()) {
		mc.ThrottledCount()
		rerr := retry.GetThrottlingError("GetManagedCluster", "client throttled", c.RetryAfterReader)
		return containerservice.ManagedCluster{}, rerr
	}

	result, rerr := c.getManagedCluster(ctx, resourceGroupName, managedClusterName)
	mc.Observe(rerr.Error())
	if rerr != nil {
		if rerr.IsThrottled() {
			// Update RetryAfterReader so that no more requests would be sent until RetryAfter expires.
			c.RetryAfterReader = rerr.RetryAfter
		}

		return result, rerr
	}

	return result, nil
}


getManagedCluster

// getManagedCluster gets a ManagedCluster.
func (c *Client) getManagedCluster(ctx context.Context, resourceGroupName string, managedClusterName string) (containerservice.ManagedCluster, *retry.Error) {
	resourceID := armclient.GetResourceID(
		c.subscriptionID,
		resourceGroupName,
		"Microsoft.ContainerService/managedClusters",
		managedClusterName,
	)
	result := containerservice.ManagedCluster{}

	response, rerr := c.armClient.GetResource(ctx, resourceID, "")
	defer c.armClient.CloseResponse(ctx, response)
	if rerr != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "managedcluster.get.request", resourceID, rerr.Error())
		return result, rerr
	}

	err := autorest.Respond(
		response,
		azure.WithErrorUnlessStatusCode(http.StatusOK),
		autorest.ByUnmarshallingJSON(&result))
	if err != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "managedcluster.get.respond", resourceID, err)
		return result, retry.GetError(response, err)
	}

	result.Response = autorest.Response{Response: response}
	return result, nil
}


List

// List gets a list of ManagedClusters in the resource group.
func (c *Client) List(ctx context.Context, resourceGroupName string) ([]containerservice.ManagedCluster, *retry.Error) {
	mc := metrics.NewMetricContext("managed_clusters", "list", resourceGroupName, c.subscriptionID, "")

	// Report errors if the client is rate limited.
	if !c.rateLimiterReader.TryAccept() {
		mc.RateLimitedCount()
		return nil, retry.GetRateLimitError(false, "ListManagedCluster")
	}

	// Report errors if the client is throttled.
	if c.RetryAfterReader.After(time.Now()) {
		mc.ThrottledCount()
		rerr := retry.GetThrottlingError("ListManagedCluster", "client throttled", c.RetryAfterReader)
		return nil, rerr
	}

	result, rerr := c.listManagedCluster(ctx, resourceGroupName)
	mc.Observe(rerr.Error())
	if rerr != nil {
		if rerr.IsThrottled() {
			// Update RetryAfterReader so that no more requests would be sent until RetryAfter expires.
			c.RetryAfterReader = rerr.RetryAfter
		}

		return result, rerr
	}

	return result, nil
}


listManagedCluster

// listManagedCluster gets a list of ManagedClusters in the resource group.
func (c *Client) listManagedCluster(ctx context.Context, resourceGroupName string) ([]containerservice.ManagedCluster, *retry.Error) {
	resourceID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.ContainerService/managedClusters",
		autorest.Encode("path", c.subscriptionID),
		autorest.Encode("path", resourceGroupName))
	result := make([]containerservice.ManagedCluster, 0)
	page := &ManagedClusterResultPage{}
	page.fn = c.listNextResults

	resp, rerr := c.armClient.GetResource(ctx, resourceID, "")
	defer c.armClient.CloseResponse(ctx, resp)
	if rerr != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "managedcluster.list.request", resourceID, rerr.Error())
		return result, rerr
	}

	var err error
	page.mclr, err = c.listResponder(resp)
	if err != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "managedcluster.list.respond", resourceID, err)
		return result, retry.GetError(resp, err)
	}

	for {
		result = append(result, page.Values()...)

		// Abort the loop when there's no nextLink in the response.
		if pointer.StringDeref(page.Response().NextLink, "") == "" {
			break
		}

		if err = page.NextWithContext(ctx); err != nil {
			klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "managedcluster.list.next", resourceID, err)
			return result, retry.GetError(page.Response().Response.Response, err)
		}
	}

	return result, nil
}


listResponder

func (c *Client) listResponder(resp *http.Response) (result containerservice.ManagedClusterListResult, err error) {
	err = autorest.Respond(
		resp,
		autorest.ByIgnoring(),
		azure.WithErrorUnlessStatusCode(http.StatusOK),
		autorest.ByUnmarshallingJSON(&result))
	result.Response = autorest.Response{Response: resp}
	return
}


managedClusterListResultPreparer

// managedClusterListResultPreparer prepares a request to retrieve the next set of results.
// It returns nil if no more results exist.
func (c *Client) managedClusterListResultPreparer(ctx context.Context, mclr containerservice.ManagedClusterListResult) (*http.Request, error) {
	if mclr.NextLink == nil || len(pointer.StringDeref(mclr.NextLink, "")) < 1 {
		return nil, nil
	}

	decorators := []autorest.PrepareDecorator{
		autorest.WithBaseURL(pointer.StringDeref(mclr.NextLink, "")),
	}
	return c.armClient.PrepareGetRequest(ctx, decorators...)
}


listNextResults

// listNextResults retrieves the next set of results, if any.
func (c *Client) listNextResults(ctx context.Context, lastResults containerservice.ManagedClusterListResult) (result containerservice.ManagedClusterListResult, err error) {
	req, err := c.managedClusterListResultPreparer(ctx, lastResults)
	if err != nil {
		return result, autorest.NewErrorWithError(err, "managedclusterclient", "listNextResults", nil, "Failure preparing next results request")
	}
	if req == nil {
		return
	}

	resp, rerr := c.armClient.Send(ctx, req)
	defer c.armClient.CloseResponse(ctx, resp)
	if rerr != nil {
		result.Response = autorest.Response{Response: resp}
		return result, autorest.NewErrorWithError(rerr.Error(), "managedclusterclient", "listNextResults", resp, "Failure sending next results request")
	}

	result, err = c.listResponder(resp)
	if err != nil {
		err = autorest.NewErrorWithError(err, "managedclusterclient", "listNextResults", resp, "Failure responding to next results request")
	}

	return
}


NextWithContext

// NextWithContext advances to the next page of values.  If there was an error making
// the request the page does not advance and the error is returned.
func (page *ManagedClusterResultPage) NextWithContext(ctx context.Context) (err error) {
	next, err := page.fn(ctx, page.mclr)
	if err != nil {
		return err
	}
	page.mclr = next
	return nil
}


Next

// Next advances to the next page of values.  If there was an error making
// the request the page does not advance and the error is returned.
// Deprecated: Use NextWithContext() instead.
func (page *ManagedClusterResultPage) Next() error {
	return page.NextWithContext(context.Background())
}


NotDone

// NotDone returns true if the page enumeration should be started or is not yet complete.
func (page ManagedClusterResultPage) NotDone() bool {
	return !page.mclr.IsEmpty()
}


Response

// Response returns the raw server response from the last page request.
func (page ManagedClusterResultPage) Response() containerservice.ManagedClusterListResult {
	return page.mclr
}


Values

// Values returns the slice of values for the current page or nil if there are no values.
func (page ManagedClusterResultPage) Values() []containerservice.ManagedCluster {
	if page.mclr.IsEmpty() {
		return nil
	}
	return *page.mclr.Value
}


CreateOrUpdate

// CreateOrUpdate creates or updates a ManagedCluster.
func (c *Client) CreateOrUpdate(ctx context.Context, resourceGroupName string, managedClusterName string, parameters containerservice.ManagedCluster, etag string) *retry.Error {
	mc := metrics.NewMetricContext("managed_clusters", "create_or_update", resourceGroupName, c.subscriptionID, "")

	// Report errors if the client is rate limited.
	if !c.rateLimiterWriter.TryAccept() {
		mc.RateLimitedCount()
		return retry.GetRateLimitError(true, "CreateOrUpdateManagedCluster")
	}

	// Report errors if the client is throttled.
	if c.RetryAfterWriter.After(time.Now()) {
		mc.ThrottledCount()
		rerr := retry.GetThrottlingError("CreateOrUpdateManagedCluster", "client throttled", c.RetryAfterWriter)
		return rerr
	}

	rerr := c.createOrUpdateManagedCluster(ctx, resourceGroupName, managedClusterName, parameters, etag)
	mc.Observe(rerr.Error())
	if rerr != nil {
		if rerr.IsThrottled() {
			// Update RetryAfterReader so that no more requests would be sent until RetryAfter expires.
			c.RetryAfterWriter = rerr.RetryAfter
		}

		return rerr
	}

	return nil
}


createOrUpdateManagedCluster

// createOrUpdateManagedCluster creates or updates a ManagedCluster.
func (c *Client) createOrUpdateManagedCluster(ctx context.Context, resourceGroupName string, managedClusterName string, parameters containerservice.ManagedCluster, etag string) *retry.Error {
	resourceID := armclient.GetResourceID(
		c.subscriptionID,
		resourceGroupName,
		"Microsoft.ContainerService/managedClusters",
		managedClusterName,
	)
	decorators := []autorest.PrepareDecorator{
		autorest.WithPathParameters("{resourceID}", map[string]interface{}{"resourceID": resourceID}),
		autorest.WithJSON(parameters),
	}
	if etag != "" {
		decorators = append(decorators, autorest.WithHeader("If-Match", autorest.String(etag)))
	}

	response, rerr := c.armClient.PutResourceWithDecorators(ctx, resourceID, parameters, decorators)
	defer c.armClient.CloseResponse(ctx, response)
	if rerr != nil {
		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "managedCluster.put.request", resourceID, rerr.Error())
		return rerr
	}

	if response != nil && response.StatusCode != http.StatusNoContent {
		_, rerr = c.createOrUpdateResponder(response)
		if rerr != nil {
			klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "managedCluster.put.respond", resourceID, rerr.Error())
			return rerr
		}
	}

	return nil
}


createOrUpdateResponder

func (c *Client) createOrUpdateResponder(resp *http.Response) (*containerservice.ManagedCluster, *retry.Error) {
	result := &containerservice.ManagedCluster{}
	err := autorest.Respond(
		resp,
		azure.WithErrorUnlessStatusCode(http.StatusOK, http.StatusCreated),
		autorest.ByUnmarshallingJSON(&result))
	result.Response = autorest.Response{Response: resp}
	return result, retry.GetError(resp, err)
}


Delete

// Delete deletes a ManagedCluster by name.
func (c *Client) Delete(ctx context.Context, resourceGroupName string, managedClusterName string) *retry.Error {
	mc := metrics.NewMetricContext("managed_clusters", "delete", resourceGroupName, c.subscriptionID, "")

	// Report errors if the client is rate limited.
	if !c.rateLimiterWriter.TryAccept() {
		mc.RateLimitedCount()
		return retry.GetRateLimitError(true, "DeleteManagedCluster")
	}

	// Report errors if the client is throttled.
	if c.RetryAfterWriter.After(time.Now()) {
		mc.ThrottledCount()
		rerr := retry.GetThrottlingError("DeleteManagedCluster", "client throttled", c.RetryAfterWriter)
		return rerr
	}

	rerr := c.deleteManagedCluster(ctx, resourceGroupName, managedClusterName)
	mc.Observe(rerr.Error())
	if rerr != nil {
		if rerr.IsThrottled() {
			// Update RetryAfterReader so that no more requests would be sent until RetryAfter expires.
			c.RetryAfterWriter = rerr.RetryAfter
		}

		return rerr
	}

	return nil
}


deleteManagedCluster

// deleteManagedCluster deletes a ManagedCluster by name.
func (c *Client) deleteManagedCluster(ctx context.Context, resourceGroupName string, managedClusterName string) *retry.Error {
	resourceID := armclient.GetResourceID(
		c.subscriptionID,
		resourceGroupName,
		"Microsoft.ContainerService/managedClusters",
		managedClusterName,
	)

	return c.armClient.DeleteResource(ctx, resourceID, "")
}


NewMockInterface

// NewMockInterface creates a new mock instance.
func NewMockInterface(ctrl *gomock.Controller) *MockInterface {
	mock := &MockInterface{ctrl: ctrl}
	mock.recorder = &MockInterfaceMockRecorder{mock}
	return mock
}


EXPECT

// EXPECT returns an object that allows the caller to indicate expected use.
func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder {
	return m.recorder
}


CreateOrUpdate

// CreateOrUpdate mocks base method.
func (m *MockInterface) CreateOrUpdate(ctx context.Context, resourceGroupName, managedClusterName string, parameters containerservice.ManagedCluster, etag string) *retry.Error {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "CreateOrUpdate", ctx, resourceGroupName, managedClusterName, parameters, etag)
	ret0, _ := ret[0].(*retry.Error)
	return ret0
}


// CreateOrUpdate indicates an expected call of CreateOrUpdate.
func (mr *MockInterfaceMockRecorder) CreateOrUpdate(ctx, resourceGroupName, managedClusterName, parameters, etag interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdate", reflect.TypeOf((*MockInterface)(nil).CreateOrUpdate), ctx, resourceGroupName, managedClusterName, parameters, etag)
}


Delete

// Delete mocks base method.
func (m *MockInterface) Delete(ctx context.Context, resourceGroupName, managedClusterName string) *retry.Error {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "Delete", ctx, resourceGroupName, managedClusterName)
	ret0, _ := ret[0].(*retry.Error)
	return ret0
}


// Delete indicates an expected call of Delete.
func (mr *MockInterfaceMockRecorder) Delete(ctx, resourceGroupName, managedClusterName interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockInterface)(nil).Delete), ctx, resourceGroupName, managedClusterName)
}


Get

// Get mocks base method.
func (m *MockInterface) Get(ctx context.Context, resourceGroupName, managedClusterName string) (containerservice.ManagedCluster, *retry.Error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "Get", ctx, resourceGroupName, managedClusterName)
	ret0, _ := ret[0].(containerservice.ManagedCluster)
	ret1, _ := ret[1].(*retry.Error)
	return ret0, ret1
}


// Get indicates an expected call of Get.
func (mr *MockInterfaceMockRecorder) Get(ctx, resourceGroupName, managedClusterName interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockInterface)(nil).Get), ctx, resourceGroupName, managedClusterName)
}


List

// List mocks base method.
func (m *MockInterface) List(ctx context.Context, resourceGroupName string) ([]containerservice.ManagedCluster, *retry.Error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "List", ctx, resourceGroupName)
	ret0, _ := ret[0].([]containerservice.ManagedCluster)
	ret1, _ := ret[1].(*retry.Error)
	return ret0, ret1
}


// List indicates an expected call of List.
func (mr *MockInterfaceMockRecorder) List(ctx, resourceGroupName interface{}) *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockInterface)(nil).List), ctx, resourceGroupName)
}