// getRegionInURL parses full resource URLS and shorter URLS
// https://www.googleapis.com/compute/v1/projects/myproject/regions/us-central1/subnetworks/a
// projects/myproject/regions/us-central1/subnetworks/a
// All return "us-central1"
func getRegionInURL(urlStr string) string {
	fields := strings.Split(urlStr, "/")
	for i, v := range fields {
		if v == "regions" && i < len(fields)-1 {
			return fields[i+1]
		}
	}
	return ""
}


func getNetworkNameViaMetadata() (string, error) {
	result, err := metadata.Get("instance/network-interfaces/0/network")
	if err != nil {
		return "", err
	}
	parts := strings.Split(result, "/")
	if len(parts) != 4 {
		return "", fmt.Errorf("unexpected response: %s", result)
	}
	return parts[3], nil
}


// getNetwork returns a GCP network
func getNetwork(svc *compute.Service, networkProjectID, networkID string) (*compute.Network, error) {
	return svc.Networks.Get(networkProjectID, networkID).Do()
}


// listSubnetworksOfNetwork returns a list of subnetworks for a particular region of a network.
func listSubnetworksOfNetwork(svc *compute.Service, networkProjectID, networkID, region string) ([]*compute.Subnetwork, error) {
	var subnets []*compute.Subnetwork
	err := svc.Subnetworks.List(networkProjectID, region).Filter(fmt.Sprintf("network eq .*/%v$", networkID)).Pages(context.Background(), func(res *compute.SubnetworkList) error {
		subnets = append(subnets, res.Items...)
		return nil
	})
	return subnets, err
}


// getProjectID returns the project's string ID given a project number or string
func getProjectID(svc *compute.Service, projectNumberOrID string) (string, error) {
	proj, err := svc.Projects.Get(projectNumberOrID).Do()
	if err != nil {
		return "", err
	}

	return proj.Name, nil
}


func getZonesForRegion(svc *compute.Service, projectID, region string) ([]string, error) {
	// TODO: use PageToken to list all not just the first 500
	listCall := svc.Zones.List(projectID)

	// Filtering by region doesn't seem to work
	// (tested in https://cloud.google.com/compute/docs/reference/latest/zones/list)
	// listCall = listCall.Filter("region eq " + region)

	var zones []string
	var accumulator = func(response *compute.ZoneList) error {
		for _, zone := range response.Items {
			regionName := lastComponent(zone.Region)
			if regionName == region {
				zones = append(zones, zone.Name)
			}
		}
		return nil
	}
	err := listCall.Pages(context.TODO(), accumulator)
	if err != nil {
		return nil, fmt.Errorf("unexpected response listing zones: %v", err)
	}
	return zones, nil
}


func findSubnetForRegion(subnetURLs []string, region string) string {
	for _, url := range subnetURLs {
		if thisRegion := getRegionInURL(url); thisRegion == region {
			return url
		}
	}
	return ""
}


func newOauthClient(tokenSource oauth2.TokenSource) (*http.Client, error) {
	if tokenSource == nil {
		var err error
		tokenSource, err = google.DefaultTokenSource(
			context.Background(),
			compute.CloudPlatformScope,
			compute.ComputeScope)
		klog.Infof("Using DefaultTokenSource %#v", tokenSource)
		if err != nil {
			return nil, err
		}
	} else {
		klog.Infof("Using existing Token Source %#v", tokenSource)
	}

	backoff := wait.Backoff{
		// These values will add up to about a minute. See #56293 for background.
		Duration: time.Second,
		Factor:   1.4,
		Steps:    10,
	}
	if err := wait.ExponentialBackoff(backoff, func() (bool, error) {
		if _, err := tokenSource.Token(); err != nil {
			klog.Errorf("error fetching initial token: %v", err)
			return false, nil
		}
		return true, nil
	}); err != nil {
		return nil, err
	}

	return oauth2.NewClient(context.Background(), tokenSource), nil
}


func (manager *gceServiceManager) getProjectsAPIEndpoint() string {
	projectsAPIEndpoint := gceComputeAPIEndpoint + "projects/"
	if manager.gce.service != nil {
		projectsAPIEndpoint = manager.gce.projectsBasePath
	}

	return projectsAPIEndpoint
}


func newAddressManager(svc CloudAddressService, serviceName, region, subnetURL, name, targetIP string, addressType cloud.LbScheme) *addressManager {
	return &addressManager{
		svc:         svc,
		logPrefix:   fmt.Sprintf("AddressManager(%q)", name),
		region:      region,
		serviceName: serviceName,
		name:        name,
		targetIP:    targetIP,
		addressType: addressType,
		tryRelease:  true,
		subnetURL:   subnetURL,
	}
}


// HoldAddress will ensure that the IP is reserved with an address - either owned by the controller
// or by a user. If the address is not the addressManager.name, then it's assumed to be a user's address.
// The string returned is the reserved IP address.
func (am *addressManager) HoldAddress() (string, error) {
	// HoldAddress starts with retrieving the address that we use for this load balancer (by name).
	// Retrieving an address by IP will indicate if the IP is reserved and if reserved by the user
	// or the controller, but won't tell us the current state of the controller's IP. The address
	// could be reserving another address; therefore, it would need to be deleted. In the normal
	// case of using a controller address, retrieving the address by name results in the fewest API
	// calls since it indicates whether a Delete is necessary before Reserve.
	klog.V(4).Infof("%v: attempting hold of IP %q Type %q", am.logPrefix, am.targetIP, am.addressType)
	// Get the address in case it was orphaned earlier
	addr, err := am.svc.GetRegionAddress(am.name, am.region)
	if err != nil && !isNotFound(err) {
		return "", err
	}

	if addr != nil {
		// If address exists, check if the address had the expected attributes.
		validationError := am.validateAddress(addr)
		if validationError == nil {
			klog.V(4).Infof("%v: address %q already reserves IP %q Type %q. No further action required.", am.logPrefix, addr.Name, addr.Address, addr.AddressType)
			return addr.Address, nil
		}

		klog.V(2).Infof("%v: deleting existing address because %v", am.logPrefix, validationError)
		err := am.svc.DeleteRegionAddress(addr.Name, am.region)
		if err != nil {
			if isNotFound(err) {
				klog.V(4).Infof("%v: address %q was not found. Ignoring.", am.logPrefix, addr.Name)
			} else {
				return "", err
			}
		} else {
			klog.V(4).Infof("%v: successfully deleted previous address %q", am.logPrefix, addr.Name)
		}
	}

	return am.ensureAddressReservation()
}


// ReleaseAddress will release the address if it's owned by the controller.
func (am *addressManager) ReleaseAddress() error {
	if !am.tryRelease {
		klog.V(4).Infof("%v: not attempting release of address %q.", am.logPrefix, am.targetIP)
		return nil
	}

	klog.V(4).Infof("%v: releasing address %q named %q", am.logPrefix, am.targetIP, am.name)
	// Controller only ever tries to unreserve the address named with the load balancer's name.
	err := am.svc.DeleteRegionAddress(am.name, am.region)
	if err != nil {
		if isNotFound(err) {
			klog.Warningf("%v: address %q was not found. Ignoring.", am.logPrefix, am.name)
			return nil
		}

		return err
	}

	klog.V(4).Infof("%v: successfully released IP %q named %q", am.logPrefix, am.targetIP, am.name)
	return nil
}


func (am *addressManager) ensureAddressReservation() (string, error) {
	// Try reserving the IP with controller-owned address name
	// If am.targetIP is an empty string, a new IP will be created.
	newAddr := &compute.Address{
		Name:        am.name,
		Description: fmt.Sprintf(`{"kubernetes.io/service-name":"%s"}`, am.serviceName),
		Address:     am.targetIP,
		AddressType: string(am.addressType),
		Subnetwork:  am.subnetURL,
	}

	reserveErr := am.svc.ReserveRegionAddress(newAddr, am.region)
	if reserveErr == nil {
		if newAddr.Address != "" {
			klog.V(4).Infof("%v: successfully reserved IP %q with name %q", am.logPrefix, newAddr.Address, newAddr.Name)
			return newAddr.Address, nil
		}

		addr, err := am.svc.GetRegionAddress(newAddr.Name, am.region)
		if err != nil {
			return "", err
		}

		klog.V(4).Infof("%v: successfully created address %q which reserved IP %q", am.logPrefix, addr.Name, addr.Address)
		return addr.Address, nil
	} else if !isHTTPErrorCode(reserveErr, http.StatusConflict) && !isHTTPErrorCode(reserveErr, http.StatusBadRequest) {
		// If the IP is already reserved:
		//    by an internal address: a StatusConflict is returned
		//    by an external address: a BadRequest is returned
		return "", reserveErr
	}

	// If the target IP was empty, we cannot try to find which IP caused a conflict.
	// If the name was already used, then the next sync will attempt deletion of that address.
	if am.targetIP == "" {
		return "", fmt.Errorf("failed to reserve address %q with no specific IP, err: %v", am.name, reserveErr)
	}

	// Reserving the address failed due to a conflict or bad request. The address manager just checked that no address
	// exists with the name, so it may belong to the user.
	addr, err := am.svc.GetRegionAddressByIP(am.region, am.targetIP)
	if err != nil {
		return "", fmt.Errorf("failed to get address by IP %q after reservation attempt, err: %q, reservation err: %q", am.targetIP, err, reserveErr)
	}

	// Check that the address attributes are as required.
	if err := am.validateAddress(addr); err != nil {
		return "", err
	}

	if am.isManagedAddress(addr) {
		// The address with this name is checked at the beginning of 'HoldAddress()', but for some reason
		// it was re-created by this point. May be possible that two controllers are running.
		klog.Warningf("%v: address %q unexpectedly existed with IP %q.", am.logPrefix, addr.Name, am.targetIP)
	} else {
		// If the retrieved address is not named with the loadbalancer name, then the controller does not own it, but will allow use of it.
		klog.V(4).Infof("%v: address %q was already reserved with name: %q, description: %q", am.logPrefix, am.targetIP, addr.Name, addr.Description)
		am.tryRelease = false
	}

	return addr.Address, nil
}


func (am *addressManager) validateAddress(addr *compute.Address) error {
	if am.targetIP != "" && am.targetIP != addr.Address {
		return fmt.Errorf("address %q does not have the expected IP %q, actual: %q", addr.Name, am.targetIP, addr.Address)
	}
	if addr.AddressType != string(am.addressType) {
		return fmt.Errorf("address %q does not have the expected address type %q, actual: %q", addr.Name, am.addressType, addr.AddressType)
	}

	return nil
}


func (am *addressManager) isManagedAddress(addr *compute.Address) bool {
	return addr.Name == am.name
}


func ensureAddressDeleted(svc CloudAddressService, name, region string) error {
	return ignoreNotFound(svc.DeleteRegionAddress(name, region))
}


func newAddressMetricContext(request, region string) *metricContext {
	return newAddressMetricContextWithVersion(request, region, computeV1Version)
}


func newAddressMetricContextWithVersion(request, region, version string) *metricContext {
	return newGenericMetricContext("address", request, region, unusedMetricLabel, version)
}


// ReserveGlobalAddress creates a global address.
// Caller is allocated a random IP if they do not specify an ipAddress. If an
// ipAddress is specified, it must belong to the current project, eg: an
// ephemeral IP associated with a global forwarding rule.
func (g *Cloud) ReserveGlobalAddress(addr *compute.Address) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newAddressMetricContext("reserve", "")
	return mc.Observe(g.c.GlobalAddresses().Insert(ctx, meta.GlobalKey(addr.Name), addr))
}


// DeleteGlobalAddress deletes a global address by name.
func (g *Cloud) DeleteGlobalAddress(name string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newAddressMetricContext("delete", "")
	return mc.Observe(g.c.GlobalAddresses().Delete(ctx, meta.GlobalKey(name)))
}


// GetGlobalAddress returns the global address by name.
func (g *Cloud) GetGlobalAddress(name string) (*compute.Address, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newAddressMetricContext("get", "")
	v, err := g.c.GlobalAddresses().Get(ctx, meta.GlobalKey(name))
	return v, mc.Observe(err)
}


// ReserveRegionAddress creates a region address
func (g *Cloud) ReserveRegionAddress(addr *compute.Address, region string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newAddressMetricContext("reserve", region)
	return mc.Observe(g.c.Addresses().Insert(ctx, meta.RegionalKey(addr.Name, region), addr))
}


// ReserveBetaRegionAddress creates a beta region address
func (g *Cloud) ReserveBetaRegionAddress(addr *computebeta.Address, region string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newAddressMetricContext("reserve", region)
	return mc.Observe(g.c.BetaAddresses().Insert(ctx, meta.RegionalKey(addr.Name, region), addr))
}


// DeleteRegionAddress deletes a region address by name.
func (g *Cloud) DeleteRegionAddress(name, region string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newAddressMetricContext("delete", region)
	return mc.Observe(g.c.Addresses().Delete(ctx, meta.RegionalKey(name, region)))
}


// GetRegionAddress returns the region address by name
func (g *Cloud) GetRegionAddress(name, region string) (*compute.Address, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newAddressMetricContext("get", region)
	v, err := g.c.Addresses().Get(ctx, meta.RegionalKey(name, region))
	return v, mc.Observe(err)
}


// GetBetaRegionAddress returns the beta region address by name
func (g *Cloud) GetBetaRegionAddress(name, region string) (*computebeta.Address, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newAddressMetricContext("get", region)
	v, err := g.c.BetaAddresses().Get(ctx, meta.RegionalKey(name, region))
	return v, mc.Observe(err)
}


// GetRegionAddressByIP returns the regional address matching the given IP address.
func (g *Cloud) GetRegionAddressByIP(region, ipAddress string) (*compute.Address, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newAddressMetricContext("list", region)
	addrs, err := g.c.Addresses().List(ctx, region, filter.Regexp("address", ipAddress))

	mc.Observe(err)
	if err != nil {
		return nil, err
	}

	if len(addrs) > 1 {
		klog.Warningf("More than one addresses matching the IP %q: %v", ipAddress, addrNames(addrs))
	}
	for _, addr := range addrs {
		if addr.Address == ipAddress {
			return addr, nil
		}
	}
	return nil, makeGoogleAPINotFoundError(fmt.Sprintf("Address with IP %q was not found in region %q", ipAddress, region))
}


// GetBetaRegionAddressByIP returns the beta regional address matching the given IP address.
func (g *Cloud) GetBetaRegionAddressByIP(region, ipAddress string) (*computebeta.Address, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newAddressMetricContext("list", region)
	addrs, err := g.c.BetaAddresses().List(ctx, region, filter.Regexp("address", ipAddress))

	mc.Observe(err)
	if err != nil {
		return nil, err
	}

	if len(addrs) > 1 {
		klog.Warningf("More than one addresses matching the IP %q: %v", ipAddress, addrNames(addrs))
	}
	for _, addr := range addrs {
		if addr.Address == ipAddress {
			return addr, nil
		}
	}
	return nil, makeGoogleAPINotFoundError(fmt.Sprintf("Address with IP %q was not found in region %q", ipAddress, region))
}


func (g *Cloud) getNetworkTierFromAddress(name, region string) (string, error) {

	addr, err := g.GetRegionAddress(name, region)
	if err != nil {
		// Can't get the network tier, just return an error.
		return "", err
	}
	return addr.NetworkTier, nil
}


func addrNames(items interface{}) []string {
	var ret []string
	switch items := items.(type) {
	case []compute.Address:
		for _, a := range items {
			ret = append(ret, a.Name)
		}
	case []computebeta.Address:
		for _, a := range items {
			ret = append(ret, a.Name)
		}
	}
	return ret
}


// Enabled returns true if the provided alpha feature is enabled
func (af *AlphaFeatureGate) Enabled(key string) bool {
	if af == nil || af.features == nil {
		return false
	}
	return af.features[key]
}


// NewAlphaFeatureGate marks the provided alpha features as enabled
func NewAlphaFeatureGate(features []string) *AlphaFeatureGate {
	featureMap := make(map[string]bool)
	for _, name := range features {
		featureMap[name] = true
	}
	return &AlphaFeatureGate{featureMap}
}


// GetLoadBalancerAnnotationType returns the type of GCP load balancer which should be assembled.
func GetLoadBalancerAnnotationType(service *v1.Service) LoadBalancerType {
	var lbType LoadBalancerType
	for _, ann := range []string{
		ServiceAnnotationLoadBalancerType,
		deprecatedServiceAnnotationLoadBalancerType,
	} {
		if v, ok := service.Annotations[ann]; ok {
			lbType = LoadBalancerType(v)
			break
		}
	}

	switch lbType {
	case LBTypeInternal, deprecatedTypeInternalLowerCase:
		return LBTypeInternal
	default:
		return lbType
	}
}


// GetLoadBalancerAnnotationBackendShare returns whether this service's backend service should be
// shared with other load balancers. Health checks and the healthcheck firewall will be shared regardless.
func GetLoadBalancerAnnotationBackendShare(service *v1.Service) bool {
	if l, exists := service.Annotations[ServiceAnnotationILBBackendShare]; exists && l == "true" {
		return true
	}

	// Check for deprecated annotation key
	if l, exists := service.Annotations[deprecatedServiceAnnotationILBBackendShare]; exists && l == "true" {
		klog.Warningf("Annotation %q is deprecated and replaced with an alpha-specific key: %q", deprecatedServiceAnnotationILBBackendShare, ServiceAnnotationILBBackendShare)
		return true
	}

	return false
}


// GetServiceNetworkTier returns the network tier of GCP load balancer
// which should be assembled, and an error if the specified tier is not
// supported.
func GetServiceNetworkTier(service *v1.Service) (cloud.NetworkTier, error) {
	l, ok := service.Annotations[NetworkTierAnnotationKey]
	if !ok {
		return cloud.NetworkTierDefault, nil
	}

	v := cloud.NetworkTier(l)
	switch v {
	case cloud.NetworkTierStandard:
		fallthrough
	case cloud.NetworkTierPremium:
		return v, nil
	default:
		return cloud.NetworkTierDefault, fmt.Errorf("unsupported network tier: %q", v)
	}
}


// GetLoadBalancerAnnotationAllowGlobalAccess returns if global access is enabled
// for the given loadbalancer service.
func GetLoadBalancerAnnotationAllowGlobalAccess(service *v1.Service) bool {
	return service.Annotations[ServiceAnnotationILBAllowGlobalAccess] == "true"
}


// GetLoadBalancerAnnotationSubnet returns the configured subnet to assign LoadBalancer IP from.
func GetLoadBalancerAnnotationSubnet(service *v1.Service) string {
	if val, exists := service.Annotations[ServiceAnnotationILBSubnet]; exists {
		return val
	}
	return ""
}


func newBackendServiceMetricContext(request, region string) *metricContext {
	return newBackendServiceMetricContextWithVersion(request, region, computeV1Version)
}


func newBackendServiceMetricContextWithVersion(request, region, version string) *metricContext {
	return newGenericMetricContext("backendservice", request, region, unusedMetricLabel, version)
}


// GetGlobalBackendService retrieves a backend by name.
func (g *Cloud) GetGlobalBackendService(name string) (*compute.BackendService, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContext("get", "")
	v, err := g.c.BackendServices().Get(ctx, meta.GlobalKey(name))
	return v, mc.Observe(err)
}


// GetBetaGlobalBackendService retrieves beta backend by name.
func (g *Cloud) GetBetaGlobalBackendService(name string) (*computebeta.BackendService, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContextWithVersion("get", "", computeBetaVersion)
	v, err := g.c.BetaBackendServices().Get(ctx, meta.GlobalKey(name))
	return v, mc.Observe(err)
}


// GetAlphaGlobalBackendService retrieves alpha backend by name.
func (g *Cloud) GetAlphaGlobalBackendService(name string) (*computealpha.BackendService, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContextWithVersion("get", "", computeAlphaVersion)
	v, err := g.c.AlphaBackendServices().Get(ctx, meta.GlobalKey(name))
	return v, mc.Observe(err)
}


// UpdateGlobalBackendService applies the given BackendService as an update to
// an existing service.
func (g *Cloud) UpdateGlobalBackendService(bg *compute.BackendService) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContext("update", "")
	return mc.Observe(g.c.BackendServices().Update(ctx, meta.GlobalKey(bg.Name), bg))
}


// UpdateBetaGlobalBackendService applies the given beta BackendService as an
// update to an existing service.
func (g *Cloud) UpdateBetaGlobalBackendService(bg *computebeta.BackendService) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContextWithVersion("update", "", computeBetaVersion)
	return mc.Observe(g.c.BetaBackendServices().Update(ctx, meta.GlobalKey(bg.Name), bg))
}


// UpdateAlphaGlobalBackendService applies the given alpha BackendService as an
// update to an existing service.
func (g *Cloud) UpdateAlphaGlobalBackendService(bg *computealpha.BackendService) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContextWithVersion("update", "", computeAlphaVersion)
	return mc.Observe(g.c.AlphaBackendServices().Update(ctx, meta.GlobalKey(bg.Name), bg))
}


// DeleteGlobalBackendService deletes the given BackendService by name.
func (g *Cloud) DeleteGlobalBackendService(name string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContext("delete", "")
	return mc.Observe(g.c.BackendServices().Delete(ctx, meta.GlobalKey(name)))
}


// CreateGlobalBackendService creates the given BackendService.
func (g *Cloud) CreateGlobalBackendService(bg *compute.BackendService) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContext("create", "")
	return mc.Observe(g.c.BackendServices().Insert(ctx, meta.GlobalKey(bg.Name), bg))
}


// CreateBetaGlobalBackendService creates the given beta BackendService.
func (g *Cloud) CreateBetaGlobalBackendService(bg *computebeta.BackendService) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContextWithVersion("create", "", computeBetaVersion)
	return mc.Observe(g.c.BetaBackendServices().Insert(ctx, meta.GlobalKey(bg.Name), bg))
}


// CreateAlphaGlobalBackendService creates the given alpha BackendService.
func (g *Cloud) CreateAlphaGlobalBackendService(bg *computealpha.BackendService) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContextWithVersion("create", "", computeAlphaVersion)
	return mc.Observe(g.c.AlphaBackendServices().Insert(ctx, meta.GlobalKey(bg.Name), bg))
}


// ListGlobalBackendServices lists all backend services in the project.
func (g *Cloud) ListGlobalBackendServices() ([]*compute.BackendService, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContext("list", "")
	v, err := g.c.BackendServices().List(ctx, filter.None)
	return v, mc.Observe(err)
}


// GetGlobalBackendServiceHealth returns the health of the BackendService
// identified by the given name, in the given instanceGroup. The
// instanceGroupLink is the fully qualified self link of an instance group.
func (g *Cloud) GetGlobalBackendServiceHealth(name string, instanceGroupLink string) (*compute.BackendServiceGroupHealth, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContext("get_health", "")
	groupRef := &compute.ResourceGroupReference{Group: instanceGroupLink}
	v, err := g.c.BackendServices().GetHealth(ctx, meta.GlobalKey(name), groupRef)
	return v, mc.Observe(err)
}


// GetRegionBackendService retrieves a backend by name.
func (g *Cloud) GetRegionBackendService(name, region string) (*compute.BackendService, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContext("get", region)
	v, err := g.c.RegionBackendServices().Get(ctx, meta.RegionalKey(name, region))
	return v, mc.Observe(err)
}


// UpdateRegionBackendService applies the given BackendService as an update to
// an existing service.
func (g *Cloud) UpdateRegionBackendService(bg *compute.BackendService, region string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContext("update", region)
	return mc.Observe(g.c.RegionBackendServices().Update(ctx, meta.RegionalKey(bg.Name, region), bg))
}


// DeleteRegionBackendService deletes the given BackendService by name.
func (g *Cloud) DeleteRegionBackendService(name, region string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContext("delete", region)
	return mc.Observe(g.c.RegionBackendServices().Delete(ctx, meta.RegionalKey(name, region)))
}


// CreateRegionBackendService creates the given BackendService.
func (g *Cloud) CreateRegionBackendService(bg *compute.BackendService, region string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContext("create", region)
	return mc.Observe(g.c.RegionBackendServices().Insert(ctx, meta.RegionalKey(bg.Name, region), bg))
}


// ListRegionBackendServices lists all backend services in the project.
func (g *Cloud) ListRegionBackendServices(region string) ([]*compute.BackendService, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContext("list", region)
	v, err := g.c.RegionBackendServices().List(ctx, region, filter.None)
	return v, mc.Observe(err)
}


// GetRegionalBackendServiceHealth returns the health of the BackendService
// identified by the given name, in the given instanceGroup. The
// instanceGroupLink is the fully qualified self link of an instance group.
func (g *Cloud) GetRegionalBackendServiceHealth(name, region string, instanceGroupLink string) (*compute.BackendServiceGroupHealth, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContext("get_health", region)
	ref := &compute.ResourceGroupReference{Group: instanceGroupLink}
	v, err := g.c.RegionBackendServices().GetHealth(ctx, meta.RegionalKey(name, region), ref)
	return v, mc.Observe(err)
}


// SetSecurityPolicyForBetaGlobalBackendService sets the given
// SecurityPolicyReference for the BackendService identified by the given name.
func (g *Cloud) SetSecurityPolicyForBetaGlobalBackendService(backendServiceName string, securityPolicyReference *computebeta.SecurityPolicyReference) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContextWithVersion("set_security_policy", "", computeBetaVersion)
	return mc.Observe(g.c.BetaBackendServices().SetSecurityPolicy(ctx, meta.GlobalKey(backendServiceName), securityPolicyReference))
}


// SetSecurityPolicyForAlphaGlobalBackendService sets the given
// SecurityPolicyReference for the BackendService identified by the given name.
func (g *Cloud) SetSecurityPolicyForAlphaGlobalBackendService(backendServiceName string, securityPolicyReference *computealpha.SecurityPolicyReference) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newBackendServiceMetricContextWithVersion("set_security_policy", "", computeAlphaVersion)
	return mc.Observe(g.c.AlphaBackendServices().SetSecurityPolicy(ctx, meta.GlobalKey(backendServiceName), securityPolicyReference))
}


func newCertMetricContext(request string) *metricContext {
	return newGenericMetricContext("cert", request, unusedMetricLabel, unusedMetricLabel, computeV1Version)
}


// GetSslCertificate returns the SslCertificate by name.
func (g *Cloud) GetSslCertificate(name string) (*compute.SslCertificate, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newCertMetricContext("get")
	v, err := g.c.SslCertificates().Get(ctx, meta.GlobalKey(name))
	return v, mc.Observe(err)
}


// CreateSslCertificate creates and returns a SslCertificate.
func (g *Cloud) CreateSslCertificate(sslCerts *compute.SslCertificate) (*compute.SslCertificate, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newCertMetricContext("create")
	err := g.c.SslCertificates().Insert(ctx, meta.GlobalKey(sslCerts.Name), sslCerts)
	if err != nil {
		return nil, mc.Observe(err)
	}
	return g.GetSslCertificate(sslCerts.Name)
}


// DeleteSslCertificate deletes the SslCertificate by name.
func (g *Cloud) DeleteSslCertificate(name string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newCertMetricContext("delete")
	return mc.Observe(g.c.SslCertificates().Delete(ctx, meta.GlobalKey(name)))
}


// ListSslCertificates lists all SslCertificates in the project.
func (g *Cloud) ListSslCertificates() ([]*compute.SslCertificate, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newCertMetricContext("list")
	v, err := g.c.SslCertificates().List(ctx, filter.None)
	return v, mc.Observe(err)
}


// Continually watches for changes to the cluster id config map
func (g *Cloud) watchClusterID(stop <-chan struct{}) {
	g.ClusterID = ClusterID{
		cfgMapKey: fmt.Sprintf("%v/%v", UIDNamespace, UIDConfigMapName),
		client:    g.client,
	}

	mapEventHandler := cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			m, ok := obj.(*v1.ConfigMap)
			if !ok || m == nil {
				klog.Errorf("Expected v1.ConfigMap, item=%+v, typeIsOk=%v", obj, ok)
				return
			}
			if m.Namespace != UIDNamespace ||
				m.Name != UIDConfigMapName {
				return
			}

			klog.V(4).Infof("Observed new configmap for clusteriD: %v, %v; setting local values", m.Name, m.Data)
			g.ClusterID.update(m)
		},
		UpdateFunc: func(old, cur interface{}) {
			m, ok := cur.(*v1.ConfigMap)
			if !ok || m == nil {
				klog.Errorf("Expected v1.ConfigMap, item=%+v, typeIsOk=%v", cur, ok)
				return
			}

			if m.Namespace != UIDNamespace ||
				m.Name != UIDConfigMapName {
				return
			}

			if reflect.DeepEqual(old, cur) {
				return
			}

			klog.V(4).Infof("Observed updated configmap for clusteriD %v, %v; setting local values", m.Name, m.Data)
			g.ClusterID.update(m)
		},
	}

	listerWatcher := cache.NewListWatchFromClient(g.ClusterID.client.CoreV1().RESTClient(), "configmaps", UIDNamespace, fields.Everything())
	var controller cache.Controller
	g.ClusterID.store, controller = cache.NewInformer(newSingleObjectListerWatcher(listerWatcher, UIDConfigMapName), &v1.ConfigMap{}, updateFuncFrequency, mapEventHandler)

	controller.Run(stop)
}


// GetID returns the id which is unique to this cluster
// if federated, return the provider id (unique to the cluster)
// if not federated, return the cluster id
func (ci *ClusterID) GetID() (string, error) {
	if err := ci.getOrInitialize(); err != nil {
		return "", err
	}

	ci.idLock.RLock()
	defer ci.idLock.RUnlock()
	if ci.clusterID == nil {
		return "", errors.New("Could not retrieve cluster id")
	}

	// If provider ID is set, (Federation is enabled) use this field
	if ci.providerID != nil {
		return *ci.providerID, nil
	}

	// providerID is not set, use the cluster id
	return *ci.clusterID, nil
}


// GetFederationID returns the id which could represent the entire Federation
// or just the cluster if not federated.
func (ci *ClusterID) GetFederationID() (string, bool, error) {
	if err := ci.getOrInitialize(); err != nil {
		return "", false, err
	}

	ci.idLock.RLock()
	defer ci.idLock.RUnlock()
	if ci.clusterID == nil {
		return "", false, errors.New("could not retrieve cluster id")
	}

	// If provider ID is not set, return false
	if ci.providerID == nil || *ci.clusterID == *ci.providerID {
		return "", false, nil
	}

	return *ci.clusterID, true, nil
}


// getOrInitialize either grabs the configmaps current value or defines the value
// and sets the configmap. This is for the case of the user calling GetClusterID()
// before the watch has begun.
func (ci *ClusterID) getOrInitialize() error {
	if ci.store == nil {
		return errors.New("Cloud.ClusterID is not ready. Call Initialize() before using")
	}

	if ci.clusterID != nil {
		return nil
	}

	exists, err := ci.getConfigMap()
	if err != nil {
		return err
	} else if exists {
		return nil
	}

	// The configmap does not exist - let's try creating one.
	newID, err := makeUID()
	if err != nil {
		return err
	}

	klog.V(4).Infof("Creating clusteriD: %v", newID)
	cfg := &v1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Name:      UIDConfigMapName,
			Namespace: UIDNamespace,
		},
	}
	cfg.Data = map[string]string{
		UIDCluster:  newID,
		UIDProvider: newID,
	}

	if _, err := ci.client.CoreV1().ConfigMaps(UIDNamespace).Create(context.TODO(), cfg, metav1.CreateOptions{}); err != nil {
		klog.Errorf("GCE cloud provider failed to create %v config map to store cluster id: %v", ci.cfgMapKey, err)
		return err
	}

	klog.V(2).Infof("Created a config map containing clusteriD: %v", newID)
	ci.update(cfg)
	return nil
}


func (ci *ClusterID) getConfigMap() (bool, error) {
	item, exists, err := ci.store.GetByKey(ci.cfgMapKey)
	if err != nil {
		return false, err
	}
	if !exists {
		return false, nil
	}

	m, ok := item.(*v1.ConfigMap)
	if !ok || m == nil {
		err = fmt.Errorf("Expected v1.ConfigMap, item=%+v, typeIsOk=%v", item, ok)
		klog.Error(err)
		return false, err
	}
	ci.update(m)
	return true, nil
}


func (ci *ClusterID) update(m *v1.ConfigMap) {
	ci.idLock.Lock()
	defer ci.idLock.Unlock()
	if clusterID, exists := m.Data[UIDCluster]; exists {
		ci.clusterID = &clusterID
	}
	if provID, exists := m.Data[UIDProvider]; exists {
		ci.providerID = &provID
	}
}


func makeUID() (string, error)

func makeUID() (string, error) {
	b := make([]byte, UIDLengthBytes)
	_, err := rand.Read(b)
	if err != nil {
		return "", err
	}
	return hex.EncodeToString(b), nil
}


func newSingleObjectListerWatcher(lw cache.ListerWatcher, objectName string) *singleObjListerWatcher {
	return &singleObjListerWatcher{lw: lw, objectName: objectName}
}


func (sow *singleObjListerWatcher) List(options metav1.ListOptions) (runtime.Object, error) {
	options.FieldSelector = "metadata.name=" + sow.objectName
	return sow.lw.List(options)
}


func (sow *singleObjListerWatcher) Watch(options metav1.ListOptions) (watch.Interface, error) {
	options.FieldSelector = "metadata.name=" + sow.objectName
	return sow.lw.Watch(options)
}


func newClustersMetricContext(request, zone string) *metricContext {
	return newGenericMetricContext("clusters", request, unusedMetricLabel, zone, computeV1Version)
}


// ListClusters will return a list of cluster names for the associated project
func (g *Cloud) ListClusters(ctx context.Context) ([]string, error) {
	allClusters := []string{}

	for _, zone := range g.managedZones {
		clusters, err := g.listClustersInZone(zone)
		if err != nil {
			return nil, err
		}
		// TODO: Scoping?  Do we need to qualify the cluster name?
		allClusters = append(allClusters, clusters...)
	}

	return allClusters, nil
}


// GetManagedClusters will return the cluster objects associated to this project
func (g *Cloud) GetManagedClusters(ctx context.Context) ([]*container.Cluster, error) {
	managedClusters := []*container.Cluster{}

	if g.regional {
		var err error
		managedClusters, err = g.getClustersInLocation(g.region)
		if err != nil {
			return nil, err
		}
	} else if len(g.managedZones) >= 1 {
		for _, zone := range g.managedZones {
			clusters, err := g.getClustersInLocation(zone)
			if err != nil {
				return nil, err
			}
			managedClusters = append(managedClusters, clusters...)
		}
	} else {
		return nil, fmt.Errorf("no zones associated with this cluster(%s)", g.ProjectID())
	}

	return managedClusters, nil
}


// Master returned the dns address of the master
func (g *Cloud) Master(ctx context.Context, clusterName string) (string, error) {
	return "k8s-" + clusterName + "-master.internal", nil
}


func (g *Cloud) listClustersInZone(zone string) ([]string, error) {
	clusters, err := g.getClustersInLocation(zone)
	if err != nil {
		return nil, err
	}

	result := []string{}
	for _, cluster := range clusters {
		result = append(result, cluster.Name)
	}
	return result, nil
}


func (g *Cloud) getClustersInLocation(zoneOrRegion string) ([]*container.Cluster, error) {
	// TODO: Issue/68913 migrate metric to list_location instead of list_zone.
	mc := newClustersMetricContext("list_zone", zoneOrRegion)
	// TODO: use PageToken to list all not just the first 500
	location := getLocationName(g.projectID, zoneOrRegion)
	list, err := g.containerService.Projects.Locations.Clusters.List(location).Do()
	if err != nil {
		return nil, mc.Observe(err)
	}
	if list.Header.Get("nextPageToken") != "" {
		klog.Errorf("Failed to get all clusters for request, received next page token %s", list.Header.Get("nextPageToken"))
	}

	return list.Clusters, mc.Observe(nil)
}


func (manager *gceServiceManager) CreateDiskOnCloudProvider(
	name string,
	sizeGb int64,
	tagsStr string,
	diskType string,
	zone string) (*Disk, error) {
	diskTypeURI, err := manager.getDiskTypeURI(
		manager.gce.region /* diskRegion */, singleZone{zone}, diskType)
	if err != nil {
		return nil, err
	}

	diskToCreateV1 := &compute.Disk{
		Name:        name,
		SizeGb:      sizeGb,
		Description: tagsStr,
		Type:        diskTypeURI,
	}

	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()
	disk := &Disk{
		ZoneInfo: singleZone{zone},
		Region:   manager.gce.region,
		Kind:     diskKind,
		Type:     diskTypeURI,
		SizeGb:   sizeGb,
	}
	return disk, manager.gce.c.Disks().Insert(ctx, meta.ZonalKey(name, zone), diskToCreateV1)
}


func (manager *gceServiceManager) CreateRegionalDiskOnCloudProvider(
	name string,
	sizeGb int64,
	tagsStr string,
	diskType string,
	replicaZones sets.String) (*Disk, error) {

	diskTypeURI, err := manager.getDiskTypeURI(
		manager.gce.region /* diskRegion */, multiZone{replicaZones}, diskType)
	if err != nil {
		return nil, err
	}
	fullyQualifiedReplicaZones := []string{}
	for _, replicaZone := range replicaZones.UnsortedList() {
		fullyQualifiedReplicaZones = append(
			fullyQualifiedReplicaZones, manager.getReplicaZoneURI(replicaZone))
	}

	diskToCreate := &compute.Disk{
		Name:         name,
		SizeGb:       sizeGb,
		Description:  tagsStr,
		Type:         diskTypeURI,
		ReplicaZones: fullyQualifiedReplicaZones,
	}

	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()
	disk := &Disk{
		ZoneInfo: multiZone{replicaZones},
		Region:   manager.gce.region,
		Name:     name,
		Kind:     diskKind,
		Type:     diskTypeURI,
		SizeGb:   sizeGb,
	}
	return disk, manager.gce.c.RegionDisks().Insert(ctx, meta.RegionalKey(name, manager.gce.region), diskToCreate)
}


func (manager *gceServiceManager) AttachDiskOnCloudProvider(
	disk *Disk,
	readWrite string,
	instanceZone string,
	instanceName string) error {
	source, err := manager.getDiskSourceURI(disk)
	if err != nil {
		return err
	}

	attachedDiskV1 := &compute.AttachedDisk{
		DeviceName: disk.Name,
		Kind:       disk.Kind,
		Mode:       readWrite,
		Source:     source,
		Type:       diskTypePersistent,
	}

	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()
	return manager.gce.c.Instances().AttachDisk(ctx, meta.ZonalKey(instanceName, instanceZone), attachedDiskV1)
}


func (manager *gceServiceManager) DetachDiskOnCloudProvider(
	instanceZone string,
	instanceName string,
	devicePath string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()
	return manager.gce.c.Instances().DetachDisk(ctx, meta.ZonalKey(instanceName, instanceZone), devicePath)
}


func (manager *gceServiceManager) GetDiskFromCloudProvider(
	zone string,
	diskName string) (*Disk, error) {
	if zone == "" {
		return nil, fmt.Errorf("can not fetch disk %q, zone is empty", diskName)
	}

	if diskName == "" {
		return nil, fmt.Errorf("can not fetch disk, zone is specified (%q), but disk name is empty", zone)
	}

	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()
	diskStable, err := manager.gce.c.Disks().Get(ctx, meta.ZonalKey(diskName, zone))
	if err != nil {
		return nil, err
	}

	zoneInfo := singleZone{strings.TrimSpace(lastComponent(diskStable.Zone))}
	if zoneInfo.zone == "" {
		zoneInfo = singleZone{zone}
	}

	region, err := manager.getRegionFromZone(zoneInfo)
	if err != nil {
		return nil, fmt.Errorf("failed to extract region from zone for %q/%q err=%v", zone, diskName, err)
	}

	return &Disk{
		ZoneInfo: zoneInfo,
		Region:   region,
		Name:     diskStable.Name,
		Kind:     diskStable.Kind,
		Type:     diskStable.Type,
		SizeGb:   diskStable.SizeGb,
	}, nil
}


func (manager *gceServiceManager) GetRegionalDiskFromCloudProvider(
	diskName string) (*Disk, error) {

	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()
	diskBeta, err := manager.gce.c.RegionDisks().Get(ctx, meta.RegionalKey(diskName, manager.gce.region))
	if err != nil {
		return nil, err
	}

	zones := sets.NewString()
	for _, zoneURI := range diskBeta.ReplicaZones {
		zones.Insert(lastComponent(zoneURI))
	}

	return &Disk{
		ZoneInfo: multiZone{zones},
		Region:   lastComponent(diskBeta.Region),
		Name:     diskBeta.Name,
		Kind:     diskBeta.Kind,
		Type:     diskBeta.Type,
		SizeGb:   diskBeta.SizeGb,
	}, nil
}


func (manager *gceServiceManager) DeleteDiskOnCloudProvider(
	zone string,
	diskName string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()
	return manager.gce.c.Disks().Delete(ctx, meta.ZonalKey(diskName, zone))
}


func (manager *gceServiceManager) DeleteRegionalDiskOnCloudProvider(
	diskName string) error {

	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()
	return manager.gce.c.RegionDisks().Delete(ctx, meta.RegionalKey(diskName, manager.gce.region))
}


func (manager *gceServiceManager) getDiskSourceURI(disk *Disk) (string, error) {
	getProjectsAPIEndpoint := manager.getProjectsAPIEndpoint()

	switch zoneInfo := disk.ZoneInfo.(type) {
	case singleZone:
		if zoneInfo.zone == "" || disk.Region == "" {
			// Unexpected, but sanity-check
			return "", fmt.Errorf("PD does not have zone/region information: %#v", disk)
		}

		return getProjectsAPIEndpoint + fmt.Sprintf(
			diskSourceURITemplateSingleZone,
			manager.gce.projectID,
			zoneInfo.zone,
			disk.Name), nil
	case multiZone:
		if zoneInfo.replicaZones == nil || zoneInfo.replicaZones.Len() <= 0 {
			// Unexpected, but sanity-check
			return "", fmt.Errorf("PD is regional but does not have any replicaZones specified: %v", disk)
		}
		return getProjectsAPIEndpoint + fmt.Sprintf(
			diskSourceURITemplateRegional,
			manager.gce.projectID,
			disk.Region,
			disk.Name), nil
	case nil:
		// Unexpected, but sanity-check
		return "", fmt.Errorf("PD did not have ZoneInfo: %v", disk)
	default:
		// Unexpected, but sanity-check
		return "", fmt.Errorf("disk.ZoneInfo has unexpected type %T", zoneInfo)
	}
}


func (manager *gceServiceManager) getDiskTypeURI(
	diskRegion string, diskZoneInfo zoneType, diskType string) (string, error) {

	getProjectsAPIEndpoint := manager.getProjectsAPIEndpoint()

	switch zoneInfo := diskZoneInfo.(type) {
	case singleZone:
		if zoneInfo.zone == "" {
			return "", fmt.Errorf("zone is empty: %v", zoneInfo)
		}

		return getProjectsAPIEndpoint + fmt.Sprintf(
			diskTypeURITemplateSingleZone,
			manager.gce.projectID,
			zoneInfo.zone,
			diskType), nil
	case multiZone:
		if zoneInfo.replicaZones == nil || zoneInfo.replicaZones.Len() <= 0 {
			return "", fmt.Errorf("zoneInfo is regional but does not have any replicaZones specified: %v", zoneInfo)
		}
		return getProjectsAPIEndpoint + fmt.Sprintf(
			diskTypeURITemplateRegional,
			manager.gce.projectID,
			diskRegion,
			diskType), nil
	case nil:
		return "", fmt.Errorf("zoneInfo nil")
	default:
		return "", fmt.Errorf("zoneInfo has unexpected type %T", zoneInfo)
	}
}


func (manager *gceServiceManager) getReplicaZoneURI(zone string) string {
	return manager.getProjectsAPIEndpoint() + fmt.Sprintf(
		replicaZoneURITemplateSingleZone,
		manager.gce.projectID,
		zone)
}


func (manager *gceServiceManager) getRegionFromZone(zoneInfo zoneType) (string, error) {
	var zone string
	switch zoneInfo := zoneInfo.(type) {
	case singleZone:
		if zoneInfo.zone == "" {
			// Unexpected, but sanity-check
			return "", fmt.Errorf("PD is single zone, but zone is not specified: %#v", zoneInfo)
		}

		zone = zoneInfo.zone
	case multiZone:
		if zoneInfo.replicaZones == nil || zoneInfo.replicaZones.Len() <= 0 {
			// Unexpected, but sanity-check
			return "", fmt.Errorf("PD is regional but does not have any replicaZones specified: %v", zoneInfo)
		}

		zone = zoneInfo.replicaZones.UnsortedList()[0]
	case nil:
		// Unexpected, but sanity-check
		return "", fmt.Errorf("zoneInfo is nil")
	default:
		// Unexpected, but sanity-check
		return "", fmt.Errorf("zoneInfo has unexpected type %T", zoneInfo)
	}

	region, err := GetGCERegion(zone)
	if err != nil {
		klog.Warningf("failed to parse GCE region from zone %q: %v", zone, err)
		region = manager.gce.region
	}

	return region, nil
}


func (manager *gceServiceManager) ResizeDiskOnCloudProvider(disk *Disk, sizeGb int64, zone string) error {
	resizeServiceRequest := &compute.DisksResizeRequest{
		SizeGb: sizeGb,
	}

	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()
	return manager.gce.c.Disks().Resize(ctx, meta.ZonalKey(disk.Name, zone), resizeServiceRequest)
}


func (manager *gceServiceManager) RegionalResizeDiskOnCloudProvider(disk *Disk, sizeGb int64) error {

	resizeServiceRequest := &compute.RegionDisksResizeRequest{
		SizeGb: sizeGb,
	}

	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()
	return manager.gce.c.RegionDisks().Resize(ctx, meta.RegionalKey(disk.Name, disk.Region), resizeServiceRequest)
}


func newDiskMetricContextZonal(request, region, zone string) *metricContext {
	return newGenericMetricContext("disk", request, region, zone, computeV1Version)
}


func newDiskMetricContextRegional(request, region string) *metricContext {
	return newGenericMetricContext("disk", request, region, unusedMetricLabel, computeV1Version)
}


// GetLabelsForVolume retrieved the label info for the provided volume
func (g *Cloud) GetLabelsForVolume(ctx context.Context, pv *v1.PersistentVolume) (map[string]string, error) {
	// Ignore any volumes that are being provisioned
	if pv.Spec.GCEPersistentDisk.PDName == cloudvolume.ProvisionedVolumeName {
		return nil, nil
	}

	// If the zone is already labeled, honor the hint
	name := pv.Spec.GCEPersistentDisk.PDName
	zone := pv.Labels[v1.LabelTopologyZone]
	if zone == "" {
		zone = pv.Labels[v1.LabelFailureDomainBetaZone]
	}

	disk, err := g.getDiskByNameAndOptionalLabelZones(name, zone)
	if err != nil {
		return nil, err
	}
	labels, err := g.GetAutoLabelsForPD(disk)
	if err != nil {
		return nil, err
	}

	return labels, nil
}


// getDiskByNameAndOptionalZone returns a Disk object for a disk (zonal or regional) for given name and (optional) zone(s) label.
func (g *Cloud) getDiskByNameAndOptionalLabelZones(name, labelZone string) (*Disk, error) {
	if labelZone == "" {
		return g.GetDiskByNameUnknownZone(name)
	}
	zoneSet, err := volumehelpers.LabelZonesToSet(labelZone)
	if err != nil {
		return nil, err
	}
	if len(zoneSet) > 1 {
		// Regional PD
		return g.getRegionalDiskByName(name)
	}
	return g.getDiskByName(name, labelZone)
}


// AttachDisk attaches given disk to the node with the specified NodeName.
// Current instance is used when instanceID is empty string.
func (g *Cloud) AttachDisk(diskName string, nodeName types.NodeName, readOnly bool, regional bool) error {
	instanceName := mapNodeNameToInstanceName(nodeName)
	instance, err := g.getInstanceByName(instanceName)
	if err != nil {
		return fmt.Errorf("error getting instance %q", instanceName)
	}

	// Try fetching as regional PD
	var disk *Disk
	var mc *metricContext
	if regional {
		disk, err = g.getRegionalDiskByName(diskName)
		if err != nil {
			return err
		}
		mc = newDiskMetricContextRegional("attach", g.region)
	} else {
		disk, err = g.getDiskByName(diskName, instance.Zone)
		if err != nil {
			return err
		}
		mc = newDiskMetricContextZonal("attach", g.region, instance.Zone)
	}

	readWrite := "READ_WRITE"
	if readOnly {
		readWrite = "READ_ONLY"
	}

	return mc.Observe(g.manager.AttachDiskOnCloudProvider(disk, readWrite, instance.Zone, instance.Name))
}


// DetachDisk detaches given disk to the node with the specified NodeName.
// Current instance is used when nodeName is empty string.
func (g *Cloud) DetachDisk(devicePath string, nodeName types.NodeName) error {
	instanceName := mapNodeNameToInstanceName(nodeName)
	inst, err := g.getInstanceByName(instanceName)
	if err != nil {
		if err == cloudprovider.InstanceNotFound {
			// If instance no longer exists, safe to assume volume is not attached.
			klog.Warningf(
				"Instance %q does not exist. DetachDisk will assume PD %q is not attached to it.",
				instanceName,
				devicePath)
			return nil
		}

		return fmt.Errorf("error getting instance %q", instanceName)
	}

	mc := newDiskMetricContextZonal("detach", g.region, inst.Zone)
	return mc.Observe(g.manager.DetachDiskOnCloudProvider(inst.Zone, inst.Name, devicePath))
}


// DiskIsAttached checks if a disk is attached to the node with the specified NodeName.
func (g *Cloud) DiskIsAttached(diskName string, nodeName types.NodeName) (bool, error) {
	instanceName := mapNodeNameToInstanceName(nodeName)
	instance, err := g.getInstanceByName(instanceName)
	if err != nil {
		if err == cloudprovider.InstanceNotFound {
			// If instance no longer exists, safe to assume volume is not attached.
			klog.Warningf(
				"Instance %q does not exist. DiskIsAttached will assume PD %q is not attached to it.",
				instanceName,
				diskName)
			return false, nil
		}

		return false, err
	}

	for _, disk := range instance.Disks {
		if disk.DeviceName == diskName {
			// Disk is still attached to node
			return true, nil
		}
	}

	return false, nil
}


// DisksAreAttached is a batch function to check if a list of disks are attached
// to the node with the specified NodeName.
func (g *Cloud) DisksAreAttached(diskNames []string, nodeName types.NodeName) (map[string]bool, error) {
	attached := make(map[string]bool)
	for _, diskName := range diskNames {
		attached[diskName] = false
	}
	instanceName := mapNodeNameToInstanceName(nodeName)
	instance, err := g.getInstanceByName(instanceName)
	if err != nil {
		if err == cloudprovider.InstanceNotFound {
			// If instance no longer exists, safe to assume volume is not attached.
			klog.Warningf(
				"Instance %q does not exist. DisksAreAttached will assume PD %v are not attached to it.",
				instanceName,
				diskNames)
			return attached, nil
		}

		return attached, err
	}

	for _, instanceDisk := range instance.Disks {
		for _, diskName := range diskNames {
			if instanceDisk.DeviceName == diskName {
				// Disk is still attached to node
				attached[diskName] = true
			}
		}
	}

	return attached, nil
}


// BulkDisksAreAttached is a batch function to check if all corresponding disks are attached to the
// nodes specified with nodeName.
func (g *Cloud) BulkDisksAreAttached(diskByNodes map[types.NodeName][]string) (map[types.NodeName]map[string]bool, error) {
	instanceNames := []string{}
	for nodeName := range diskByNodes {
		instanceNames = append(instanceNames, mapNodeNameToInstanceName(nodeName))
	}

	// List all instances with the given instance names
	// Then for each instance listed, add the disks attached to that instance to a map
	listedInstances, err := g.getFoundInstanceByNames(instanceNames)
	if err != nil {
		return nil, fmt.Errorf("error listing instances: %v", err)
	}
	listedInstanceNamesToDisks := make(map[string][]*compute.AttachedDisk)
	for _, instance := range listedInstances {
		listedInstanceNamesToDisks[instance.Name] = instance.Disks
	}

	verifyDisksAttached := make(map[types.NodeName]map[string]bool)

	// For each node and its desired attached disks that needs to be verified
	for nodeName, disksToVerify := range diskByNodes {
		instanceName := canonicalizeInstanceName(mapNodeNameToInstanceName(nodeName))
		disksActuallyAttached := listedInstanceNamesToDisks[instanceName]
		verifyDisksAttached[nodeName] = verifyDisksAttachedToNode(disksToVerify, disksActuallyAttached)
	}

	return verifyDisksAttached, nil
}


func (g *Cloud) CreateDisk(

// CreateDisk creates a new Persistent Disk, with the specified name &
// size, in the specified zone. It stores specified tags encoded in
// JSON in Description field.
func (g *Cloud) CreateDisk(
	name string, diskType string, zone string, sizeGb int64, tags map[string]string) (*Disk, error) {
	// Do not allow creation of PDs in zones that are do not have nodes. Such PDs
	// are not currently usable.
	curZones, err := g.GetAllCurrentZones()
	if err != nil {
		return nil, err
	}
	if !curZones.Has(zone) {
		return nil, fmt.Errorf("kubernetes does not have a node in zone %q", zone)
	}

	tagsStr, err := g.encodeDiskTags(tags)
	if err != nil {
		return nil, err
	}

	diskType, err = getDiskType(diskType)
	if err != nil {
		return nil, err
	}

	mc := newDiskMetricContextZonal("create", g.region, zone)
	disk, err := g.manager.CreateDiskOnCloudProvider(
		name, sizeGb, tagsStr, diskType, zone)

	mc.Observe(err)
	if err != nil {
		if isGCEError(err, "alreadyExists") {
			klog.Warningf("GCE PD %q already exists, reusing", name)
			return g.manager.GetDiskFromCloudProvider(zone, name)
		}
		return nil, err
	}
	return disk, nil
}


func (g *Cloud) CreateRegionalDisk(

// CreateRegionalDisk creates a new Regional Persistent Disk, with the specified
// name & size, replicated to the specified zones. It stores specified tags
// encoded in JSON in Description field.
func (g *Cloud) CreateRegionalDisk(
	name string, diskType string, replicaZones sets.String, sizeGb int64, tags map[string]string) (*Disk, error) {

	// Do not allow creation of PDs in zones that are do not have nodes. Such PDs
	// are not currently usable. This functionality should be reverted to checking
	// against managed zones if we want users to be able to create RegionalDisks
	// in zones where there are no nodes
	curZones, err := g.GetAllCurrentZones()
	if err != nil {
		return nil, err
	}
	if !curZones.IsSuperset(replicaZones) {
		return nil, fmt.Errorf("kubernetes does not have nodes in specified zones: %q. Zones that contain nodes: %q", replicaZones.Difference(curZones), curZones)
	}

	tagsStr, err := g.encodeDiskTags(tags)
	if err != nil {
		return nil, err
	}

	diskType, err = getDiskType(diskType)
	if err != nil {
		return nil, err
	}

	mc := newDiskMetricContextRegional("create", g.region)

	disk, err := g.manager.CreateRegionalDiskOnCloudProvider(
		name, sizeGb, tagsStr, diskType, replicaZones)

	mc.Observe(err)
	if err != nil {
		if isGCEError(err, "alreadyExists") {
			klog.Warningf("GCE PD %q already exists, reusing", name)
			return g.manager.GetRegionalDiskFromCloudProvider(name)
		}
		return nil, err
	}
	return disk, nil
}


func getDiskType(diskType string) (string, error) {
	switch diskType {
	case DiskTypeSSD, DiskTypeStandard:
		return diskType, nil
	case "":
		return diskTypeDefault, nil
	default:
		return "", fmt.Errorf("invalid GCE disk type %q", diskType)
	}
}


// DeleteDisk deletes rgw referenced persistent disk.
func (g *Cloud) DeleteDisk(diskToDelete string) error {
	err := g.doDeleteDisk(diskToDelete)
	if isGCEError(err, "resourceInUseByAnotherResource") {
		return volerr.NewDeletedVolumeInUseError(err.Error())
	}

	if err == cloudprovider.DiskNotFound {
		return nil
	}
	return err
}


// ResizeDisk expands given disk and returns new disk size
func (g *Cloud) ResizeDisk(diskToResize string, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) {
	disk, err := g.GetDiskByNameUnknownZone(diskToResize)
	if err != nil {
		return oldSize, err
	}

	// GCE resizes in chunks of GiBs
	requestGIB, err := volumehelpers.RoundUpToGiB(newSize)
	if err != nil {
		return oldSize, err
	}

	newSizeQuant := resource.MustParse(fmt.Sprintf("%dGi", requestGIB))

	// If disk is already of size equal or greater than requested size, we simply return
	if disk.SizeGb >= requestGIB {
		return newSizeQuant, nil
	}

	var mc *metricContext

	switch zoneInfo := disk.ZoneInfo.(type) {
	case singleZone:
		mc = newDiskMetricContextZonal("resize", disk.Region, zoneInfo.zone)
		err := g.manager.ResizeDiskOnCloudProvider(disk, requestGIB, zoneInfo.zone)

		if err != nil {
			return oldSize, mc.Observe(err)
		}
		return newSizeQuant, mc.Observe(err)
	case multiZone:
		mc = newDiskMetricContextRegional("resize", disk.Region)
		err := g.manager.RegionalResizeDiskOnCloudProvider(disk, requestGIB)

		if err != nil {
			return oldSize, mc.Observe(err)
		}
		return newSizeQuant, mc.Observe(err)
	case nil:
		return oldSize, fmt.Errorf("PD has nil ZoneInfo: %v", disk)
	default:
		return oldSize, fmt.Errorf("disk.ZoneInfo has unexpected type %T", zoneInfo)
	}
}


// GetAutoLabelsForPD builds the labels that should be automatically added to a PersistentVolume backed by a GCE PD
// Specifically, this builds Topology (zone) and Region labels.
// The PersistentVolumeLabel admission controller calls this and adds the labels when a PV is created.
func (g *Cloud) GetAutoLabelsForPD(disk *Disk) (map[string]string, error) {
	labels := make(map[string]string)
	switch zoneInfo := disk.ZoneInfo.(type) {
	case singleZone:
		if zoneInfo.zone == "" || disk.Region == "" {
			// Unexpected, but sanity-check
			return nil, fmt.Errorf("PD did not have zone/region information: %v", disk)
		}
		labels[v1.LabelTopologyZone] = zoneInfo.zone
		labels[v1.LabelTopologyRegion] = disk.Region
	case multiZone:
		if zoneInfo.replicaZones == nil || zoneInfo.replicaZones.Len() <= 0 {
			// Unexpected, but sanity-check
			return nil, fmt.Errorf("PD is regional but does not have any replicaZones specified: %v", disk)
		}
		labels[v1.LabelTopologyZone] =
			volumehelpers.ZonesSetToLabelValue(zoneInfo.replicaZones)
		labels[v1.LabelTopologyRegion] = disk.Region
	case nil:
		// Unexpected, but sanity-check
		return nil, fmt.Errorf("PD did not have ZoneInfo: %v", disk)
	default:
		// Unexpected, but sanity-check
		return nil, fmt.Errorf("disk.ZoneInfo has unexpected type %T", zoneInfo)
	}

	return labels, nil
}


// Returns a Disk for the disk, if it is found in the specified zone.
// If not found, returns (nil, nil)
func (g *Cloud) findDiskByName(diskName string, zone string) (*Disk, error) {
	mc := newDiskMetricContextZonal("get", g.region, zone)
	disk, err := g.manager.GetDiskFromCloudProvider(zone, diskName)
	if err == nil {
		return disk, mc.Observe(nil)
	}
	if !isHTTPErrorCode(err, http.StatusNotFound) {
		return nil, mc.Observe(err)
	}
	return nil, mc.Observe(nil)
}


// Like findDiskByName, but returns an error if the disk is not found
func (g *Cloud) getDiskByName(diskName string, zone string) (*Disk, error) {
	disk, err := g.findDiskByName(diskName, zone)
	if disk == nil && err == nil {
		return nil, fmt.Errorf("GCE persistent disk not found: diskName=%q zone=%q", diskName, zone)
	}
	return disk, err
}


// Returns a Disk for the regional disk, if it is found.
// If not found, returns (nil, nil)
func (g *Cloud) findRegionalDiskByName(diskName string) (*Disk, error) {
	mc := newDiskMetricContextRegional("get", g.region)
	disk, err := g.manager.GetRegionalDiskFromCloudProvider(diskName)
	if err == nil {
		return disk, mc.Observe(nil)
	}
	if !isHTTPErrorCode(err, http.StatusNotFound) {
		return nil, mc.Observe(err)
	}
	return nil, mc.Observe(nil)
}


// Like findRegionalDiskByName, but returns an error if the disk is not found
func (g *Cloud) getRegionalDiskByName(diskName string) (*Disk, error) {
	disk, err := g.findRegionalDiskByName(diskName)
	if disk == nil && err == nil {
		return nil, fmt.Errorf("GCE regional persistent disk not found: diskName=%q", diskName)
	}
	return disk, err
}


// GetDiskByNameUnknownZone scans all managed zones to return the GCE PD
// Prefer getDiskByName, if the zone can be established
// Return cloudprovider.DiskNotFound if the given disk cannot be found in any zone
func (g *Cloud) GetDiskByNameUnknownZone(diskName string) (*Disk, error) {
	regionalDisk, err := g.getRegionalDiskByName(diskName)
	if err == nil {
		return regionalDisk, err
	}

	// Note: this is the gotcha right now with GCE PD support:
	// disk names are not unique per-region.
	// (I can create two volumes with name "myvol" in e.g. us-central1-b & us-central1-f)
	// For now, this is simply undefined behaviour.
	//
	// In future, we will have to require users to qualify their disk
	// "us-central1-a/mydisk".  We could do this for them as part of
	// admission control, but that might be a little weird (values changing
	// on create)

	var found *Disk
	for _, zone := range g.managedZones {
		disk, err := g.findDiskByName(diskName, zone)
		if err != nil {
			return nil, err
		}
		// findDiskByName returns (nil,nil) if the disk doesn't exist, so we can't
		// assume that a disk was found unless disk is non-nil.
		if disk == nil {
			continue
		}
		if found != nil {
			switch zoneInfo := disk.ZoneInfo.(type) {
			case multiZone:
				if zoneInfo.replicaZones.Has(zone) {
					klog.Warningf("GCE PD name (%q) was found in multiple zones (%q), but ok because it is a RegionalDisk.",
						diskName, zoneInfo.replicaZones)
					continue
				}
				return nil, fmt.Errorf("GCE PD name was found in multiple zones: %q", diskName)
			default:
				return nil, fmt.Errorf("GCE PD name was found in multiple zones: %q", diskName)
			}
		}
		found = disk
	}
	if found != nil {
		return found, nil
	}
	klog.Warningf("GCE persistent disk %q not found in managed zones (%s)",
		diskName, strings.Join(g.managedZones, ","))

	return nil, cloudprovider.DiskNotFound
}


// encodeDiskTags encodes requested volume tags into JSON string, as GCE does
// not support tags on GCE PDs and we use Description field as fallback.
func (g *Cloud) encodeDiskTags(tags map[string]string) (string, error) {
	if len(tags) == 0 {
		// No tags -> empty JSON
		return "", nil
	}

	enc, err := json.Marshal(tags)
	if err != nil {
		return "", err
	}
	return string(enc), nil
}


func (g *Cloud) doDeleteDisk(diskToDelete string) error {
	disk, err := g.GetDiskByNameUnknownZone(diskToDelete)
	if err != nil {
		return err
	}

	var mc *metricContext

	switch zoneInfo := disk.ZoneInfo.(type) {
	case singleZone:
		mc = newDiskMetricContextZonal("delete", disk.Region, zoneInfo.zone)
		return mc.Observe(g.manager.DeleteDiskOnCloudProvider(zoneInfo.zone, disk.Name))
	case multiZone:
		mc = newDiskMetricContextRegional("delete", disk.Region)
		return mc.Observe(g.manager.DeleteRegionalDiskOnCloudProvider(disk.Name))
	case nil:
		return fmt.Errorf("PD has nil ZoneInfo: %v", disk)
	default:
		return fmt.Errorf("disk.ZoneInfo has unexpected type %T", zoneInfo)
	}
}


// isGCEError returns true if given error is a googleapi.Error with given
// reason (e.g. "resourceInUseByAnotherResource")
func isGCEError(err error, reason string) bool {
	apiErr, ok := err.(*googleapi.Error)
	if !ok {
		return false
	}

	for _, e := range apiErr.Errors {
		if e.Reason == reason {
			return true
		}
	}
	return false
}


// verifyDisksAttachedToNode takes in an slice of disks that should be attached to an instance, and the
// slice of disks actually attached to it. It returns a map verifying if the disks are actually attached.
func verifyDisksAttachedToNode(disksToVerify []string, disksActuallyAttached []*compute.AttachedDisk) map[string]bool {
	verifiedDisks := make(map[string]bool)
	diskNamesActuallyAttached := sets.NewString()
	for _, disk := range disksActuallyAttached {
		diskNamesActuallyAttached.Insert(disk.DeviceName)
	}

	// For every disk that's supposed to be attached, verify that it is
	for _, diskName := range disksToVerify {
		verifiedDisks[diskName] = diskNamesActuallyAttached.Has(diskName)
	}

	return verifiedDisks
}


func newFirewallMetricContext(request string) *metricContext {
	return newGenericMetricContext("firewall", request, unusedMetricLabel, unusedMetricLabel, computeV1Version)
}


// GetFirewall returns the Firewall by name.
func (g *Cloud) GetFirewall(name string) (*compute.Firewall, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newFirewallMetricContext("get")
	v, err := g.c.Firewalls().Get(ctx, meta.GlobalKey(name))
	return v, mc.Observe(err)
}


// CreateFirewall creates the passed firewall
func (g *Cloud) CreateFirewall(f *compute.Firewall) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newFirewallMetricContext("create")
	return mc.Observe(g.c.Firewalls().Insert(ctx, meta.GlobalKey(f.Name), f))
}


// DeleteFirewall deletes the given firewall rule.
func (g *Cloud) DeleteFirewall(name string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newFirewallMetricContext("delete")
	return mc.Observe(g.c.Firewalls().Delete(ctx, meta.GlobalKey(name)))
}


// UpdateFirewall applies the given firewall as an update to an existing service.
func (g *Cloud) UpdateFirewall(f *compute.Firewall) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newFirewallMetricContext("update")
	return mc.Observe(g.c.Firewalls().Update(ctx, meta.GlobalKey(f.Name), f))
}


// PatchFirewall applies the given firewall as an update to an existing service.
func (g *Cloud) PatchFirewall(f *compute.Firewall) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newFirewallMetricContext("Patch")
	return mc.Observe(g.c.Firewalls().Patch(ctx, meta.GlobalKey(f.Name), f))
}


func newForwardingRuleMetricContext(request, region string) *metricContext {
	return newForwardingRuleMetricContextWithVersion(request, region, computeV1Version)
}


func newForwardingRuleMetricContextWithVersion(request, region, version string) *metricContext {
	return newGenericMetricContext("forwardingrule", request, region, unusedMetricLabel, version)
}


// CreateGlobalForwardingRule creates the passed GlobalForwardingRule
func (g *Cloud) CreateGlobalForwardingRule(rule *compute.ForwardingRule) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newForwardingRuleMetricContext("create", "")
	return mc.Observe(g.c.GlobalForwardingRules().Insert(ctx, meta.GlobalKey(rule.Name), rule))
}


// SetProxyForGlobalForwardingRule links the given TargetHttp(s)Proxy with the given GlobalForwardingRule.
// targetProxyLink is the SelfLink of a TargetHttp(s)Proxy.
func (g *Cloud) SetProxyForGlobalForwardingRule(forwardingRuleName, targetProxyLink string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newForwardingRuleMetricContext("set_proxy", "")
	target := &compute.TargetReference{Target: targetProxyLink}
	return mc.Observe(g.c.GlobalForwardingRules().SetTarget(ctx, meta.GlobalKey(forwardingRuleName), target))
}


// DeleteGlobalForwardingRule deletes the GlobalForwardingRule by name.
func (g *Cloud) DeleteGlobalForwardingRule(name string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newForwardingRuleMetricContext("delete", "")
	return mc.Observe(g.c.GlobalForwardingRules().Delete(ctx, meta.GlobalKey(name)))
}


// GetGlobalForwardingRule returns the GlobalForwardingRule by name.
func (g *Cloud) GetGlobalForwardingRule(name string) (*compute.ForwardingRule, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newForwardingRuleMetricContext("get", "")
	v, err := g.c.GlobalForwardingRules().Get(ctx, meta.GlobalKey(name))
	return v, mc.Observe(err)
}


// ListGlobalForwardingRules lists all GlobalForwardingRules in the project.
func (g *Cloud) ListGlobalForwardingRules() ([]*compute.ForwardingRule, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newForwardingRuleMetricContext("list", "")
	v, err := g.c.GlobalForwardingRules().List(ctx, filter.None)
	return v, mc.Observe(err)
}


// GetRegionForwardingRule returns the RegionalForwardingRule by name & region.
func (g *Cloud) GetRegionForwardingRule(name, region string) (*compute.ForwardingRule, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newForwardingRuleMetricContext("get", region)
	v, err := g.c.ForwardingRules().Get(ctx, meta.RegionalKey(name, region))
	return v, mc.Observe(err)
}


// GetAlphaRegionForwardingRule returns the Alpha forwarding rule by name & region.
func (g *Cloud) GetAlphaRegionForwardingRule(name, region string) (*computealpha.ForwardingRule, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newForwardingRuleMetricContextWithVersion("get", region, computeAlphaVersion)
	v, err := g.c.AlphaForwardingRules().Get(ctx, meta.RegionalKey(name, region))
	return v, mc.Observe(err)
}


// GetBetaRegionForwardingRule returns the Beta forwarding rule by name & region.
func (g *Cloud) GetBetaRegionForwardingRule(name, region string) (*computebeta.ForwardingRule, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newForwardingRuleMetricContextWithVersion("get", region, computeBetaVersion)
	v, err := g.c.BetaForwardingRules().Get(ctx, meta.RegionalKey(name, region))
	return v, mc.Observe(err)
}


// ListRegionForwardingRules lists all RegionalForwardingRules in the project & region.
func (g *Cloud) ListRegionForwardingRules(region string) ([]*compute.ForwardingRule, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newForwardingRuleMetricContext("list", region)
	v, err := g.c.ForwardingRules().List(ctx, region, filter.None)
	return v, mc.Observe(err)
}


// ListAlphaRegionForwardingRules lists all RegionalForwardingRules in the project & region.
func (g *Cloud) ListAlphaRegionForwardingRules(region string) ([]*computealpha.ForwardingRule, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newForwardingRuleMetricContextWithVersion("list", region, computeAlphaVersion)
	v, err := g.c.AlphaForwardingRules().List(ctx, region, filter.None)
	return v, mc.Observe(err)
}


// ListBetaRegionForwardingRules lists all RegionalForwardingRules in the project & region.
func (g *Cloud) ListBetaRegionForwardingRules(region string) ([]*computebeta.ForwardingRule, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newForwardingRuleMetricContextWithVersion("list", region, computeBetaVersion)
	v, err := g.c.BetaForwardingRules().List(ctx, region, filter.None)
	return v, mc.Observe(err)
}


// CreateRegionForwardingRule creates and returns a
// RegionalForwardingRule that points to the given BackendService
func (g *Cloud) CreateRegionForwardingRule(rule *compute.ForwardingRule, region string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newForwardingRuleMetricContext("create", region)
	return mc.Observe(g.c.ForwardingRules().Insert(ctx, meta.RegionalKey(rule.Name, region), rule))
}


// CreateAlphaRegionForwardingRule creates and returns an Alpha
// forwarding rule in the given region.
func (g *Cloud) CreateAlphaRegionForwardingRule(rule *computealpha.ForwardingRule, region string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newForwardingRuleMetricContextWithVersion("create", region, computeAlphaVersion)
	return mc.Observe(g.c.AlphaForwardingRules().Insert(ctx, meta.RegionalKey(rule.Name, region), rule))
}


// CreateBetaRegionForwardingRule creates and returns a Beta
// forwarding rule in the given region.
func (g *Cloud) CreateBetaRegionForwardingRule(rule *computebeta.ForwardingRule, region string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newForwardingRuleMetricContextWithVersion("create", region, computeBetaVersion)
	return mc.Observe(g.c.BetaForwardingRules().Insert(ctx, meta.RegionalKey(rule.Name, region), rule))
}


// DeleteRegionForwardingRule deletes the RegionalForwardingRule by name & region.
func (g *Cloud) DeleteRegionForwardingRule(name, region string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newForwardingRuleMetricContext("delete", region)
	return mc.Observe(g.c.ForwardingRules().Delete(ctx, meta.RegionalKey(name, region)))
}


func (g *Cloud) getNetworkTierFromForwardingRule(name, region string) (string, error) {
	fwdRule, err := g.GetRegionForwardingRule(name, region)
	if err != nil {
		// Can't get the network tier, just return an error.
		return "", err
	}
	return fwdRule.NetworkTier, nil
}


func init()

func init() {
	if v, err := utilversion.ParseGeneric("1.7.2"); err != nil {
		klog.Fatalf("Failed to parse version for minNodesHealthCheckVersion: %v", err)
	} else {
		minNodesHealthCheckVersion = v
	}
}


func newHealthcheckMetricContext(request string) *metricContext {
	return newHealthcheckMetricContextWithVersion(request, computeV1Version)
}


func newHealthcheckMetricContextWithVersion(request, version string) *metricContext {
	return newGenericMetricContext("healthcheck", request, unusedMetricLabel, unusedMetricLabel, version)
}


// GetHTTPHealthCheck returns the given HttpHealthCheck by name.
func (g *Cloud) GetHTTPHealthCheck(name string) (*compute.HttpHealthCheck, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContext("get_legacy")
	v, err := g.c.HttpHealthChecks().Get(ctx, meta.GlobalKey(name))
	return v, mc.Observe(err)
}


// UpdateHTTPHealthCheck applies the given HttpHealthCheck as an update.
func (g *Cloud) UpdateHTTPHealthCheck(hc *compute.HttpHealthCheck) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContext("update_legacy")
	return mc.Observe(g.c.HttpHealthChecks().Update(ctx, meta.GlobalKey(hc.Name), hc))
}


// DeleteHTTPHealthCheck deletes the given HttpHealthCheck by name.
func (g *Cloud) DeleteHTTPHealthCheck(name string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContext("delete_legacy")
	return mc.Observe(g.c.HttpHealthChecks().Delete(ctx, meta.GlobalKey(name)))
}


// CreateHTTPHealthCheck creates the given HttpHealthCheck.
func (g *Cloud) CreateHTTPHealthCheck(hc *compute.HttpHealthCheck) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContext("create_legacy")
	return mc.Observe(g.c.HttpHealthChecks().Insert(ctx, meta.GlobalKey(hc.Name), hc))
}


// ListHTTPHealthChecks lists all HttpHealthChecks in the project.
func (g *Cloud) ListHTTPHealthChecks() ([]*compute.HttpHealthCheck, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContext("list_legacy")
	v, err := g.c.HttpHealthChecks().List(ctx, filter.None)
	return v, mc.Observe(err)
}


// GetHTTPSHealthCheck returns the given HttpsHealthCheck by name.
func (g *Cloud) GetHTTPSHealthCheck(name string) (*compute.HttpsHealthCheck, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContext("get_legacy")
	v, err := g.c.HttpsHealthChecks().Get(ctx, meta.GlobalKey(name))
	return v, mc.Observe(err)
}


// UpdateHTTPSHealthCheck applies the given HttpsHealthCheck as an update.
func (g *Cloud) UpdateHTTPSHealthCheck(hc *compute.HttpsHealthCheck) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContext("update_legacy")
	return mc.Observe(g.c.HttpsHealthChecks().Update(ctx, meta.GlobalKey(hc.Name), hc))
}


// DeleteHTTPSHealthCheck deletes the given HttpsHealthCheck by name.
func (g *Cloud) DeleteHTTPSHealthCheck(name string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContext("delete_legacy")
	return mc.Observe(g.c.HttpsHealthChecks().Delete(ctx, meta.GlobalKey(name)))
}


// CreateHTTPSHealthCheck creates the given HttpsHealthCheck.
func (g *Cloud) CreateHTTPSHealthCheck(hc *compute.HttpsHealthCheck) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContext("create_legacy")
	return mc.Observe(g.c.HttpsHealthChecks().Insert(ctx, meta.GlobalKey(hc.Name), hc))
}


// ListHTTPSHealthChecks lists all HttpsHealthChecks in the project.
func (g *Cloud) ListHTTPSHealthChecks() ([]*compute.HttpsHealthCheck, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContext("list_legacy")
	v, err := g.c.HttpsHealthChecks().List(ctx, filter.None)
	return v, mc.Observe(err)
}


// GetHealthCheck returns the given HealthCheck by name.
func (g *Cloud) GetHealthCheck(name string) (*compute.HealthCheck, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContext("get")
	v, err := g.c.HealthChecks().Get(ctx, meta.GlobalKey(name))
	return v, mc.Observe(err)
}


// GetAlphaHealthCheck returns the given alpha HealthCheck by name.
func (g *Cloud) GetAlphaHealthCheck(name string) (*computealpha.HealthCheck, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContextWithVersion("get", computeAlphaVersion)
	v, err := g.c.AlphaHealthChecks().Get(ctx, meta.GlobalKey(name))
	return v, mc.Observe(err)
}


// GetBetaHealthCheck returns the given beta HealthCheck by name.
func (g *Cloud) GetBetaHealthCheck(name string) (*computebeta.HealthCheck, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContextWithVersion("get", computeBetaVersion)
	v, err := g.c.BetaHealthChecks().Get(ctx, meta.GlobalKey(name))
	return v, mc.Observe(err)
}


// UpdateHealthCheck applies the given HealthCheck as an update.
func (g *Cloud) UpdateHealthCheck(hc *compute.HealthCheck) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContext("update")
	return mc.Observe(g.c.HealthChecks().Update(ctx, meta.GlobalKey(hc.Name), hc))
}


// UpdateAlphaHealthCheck applies the given alpha HealthCheck as an update.
func (g *Cloud) UpdateAlphaHealthCheck(hc *computealpha.HealthCheck) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContextWithVersion("update", computeAlphaVersion)
	return mc.Observe(g.c.AlphaHealthChecks().Update(ctx, meta.GlobalKey(hc.Name), hc))
}


// UpdateBetaHealthCheck applies the given beta HealthCheck as an update.
func (g *Cloud) UpdateBetaHealthCheck(hc *computebeta.HealthCheck) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContextWithVersion("update", computeBetaVersion)
	return mc.Observe(g.c.BetaHealthChecks().Update(ctx, meta.GlobalKey(hc.Name), hc))
}


// DeleteHealthCheck deletes the given HealthCheck by name.
func (g *Cloud) DeleteHealthCheck(name string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContext("delete")
	return mc.Observe(g.c.HealthChecks().Delete(ctx, meta.GlobalKey(name)))
}


// CreateHealthCheck creates the given HealthCheck.
func (g *Cloud) CreateHealthCheck(hc *compute.HealthCheck) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContext("create")
	return mc.Observe(g.c.HealthChecks().Insert(ctx, meta.GlobalKey(hc.Name), hc))
}


// CreateAlphaHealthCheck creates the given alpha HealthCheck.
func (g *Cloud) CreateAlphaHealthCheck(hc *computealpha.HealthCheck) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContextWithVersion("create", computeAlphaVersion)
	return mc.Observe(g.c.AlphaHealthChecks().Insert(ctx, meta.GlobalKey(hc.Name), hc))
}


// CreateBetaHealthCheck creates the given beta HealthCheck.
func (g *Cloud) CreateBetaHealthCheck(hc *computebeta.HealthCheck) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContextWithVersion("create", computeBetaVersion)
	return mc.Observe(g.c.BetaHealthChecks().Insert(ctx, meta.GlobalKey(hc.Name), hc))
}


// ListHealthChecks lists all HealthCheck in the project.
func (g *Cloud) ListHealthChecks() ([]*compute.HealthCheck, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newHealthcheckMetricContext("list")
	v, err := g.c.HealthChecks().List(ctx, filter.None)
	return v, mc.Observe(err)
}


func GetNodesHealthCheckPort() int32

// GetNodesHealthCheckPort returns the health check port used by the GCE load
// balancers (l4) for performing health checks on nodes.
func GetNodesHealthCheckPort() int32 {
	return lbNodesHealthCheckPort
}


func GetNodesHealthCheckPath() string

// GetNodesHealthCheckPath returns the health check path used by the GCE load
// balancers (l4) for performing health checks on nodes.
func GetNodesHealthCheckPath() string {
	return nodesHealthCheckPath
}


// isAtLeastMinNodesHealthCheckVersion checks if a version is higher than
// `minNodesHealthCheckVersion`.
func isAtLeastMinNodesHealthCheckVersion(vstring string) bool {
	version, err := utilversion.ParseGeneric(vstring)
	if err != nil {
		klog.Errorf("vstring (%s) is not a valid version string: %v", vstring, err)
		return false
	}
	return version.AtLeast(minNodesHealthCheckVersion)
}


// supportsNodesHealthCheck returns false if anyone of the nodes has version
// lower than `minNodesHealthCheckVersion`.
func supportsNodesHealthCheck(nodes []*v1.Node) bool {
	for _, node := range nodes {
		if !isAtLeastMinNodesHealthCheckVersion(node.Status.NodeInfo.KubeProxyVersion) {
			return false
		}
	}
	return true
}


func newInstanceGroupMetricContext(request string, zone string) *metricContext {
	return newGenericMetricContext("instancegroup", request, unusedMetricLabel, zone, computeV1Version)
}


// CreateInstanceGroup creates an instance group with the given
// instances. It is the callers responsibility to add named ports.
func (g *Cloud) CreateInstanceGroup(ig *compute.InstanceGroup, zone string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newInstanceGroupMetricContext("create", zone)
	return mc.Observe(g.c.InstanceGroups().Insert(ctx, meta.ZonalKey(ig.Name, zone), ig))
}


// DeleteInstanceGroup deletes an instance group.
func (g *Cloud) DeleteInstanceGroup(name string, zone string) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newInstanceGroupMetricContext("delete", zone)
	return mc.Observe(g.c.InstanceGroups().Delete(ctx, meta.ZonalKey(name, zone)))
}


// FilterInstanceGroupsByName lists all InstanceGroups in the project and
// zone that match the name regexp.
func (g *Cloud) FilterInstanceGroupsByNamePrefix(namePrefix, zone string) ([]*compute.InstanceGroup, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()
	mc := newInstanceGroupMetricContext("filter", zone)
	v, err := g.c.InstanceGroups().List(ctx, zone, filter.Regexp("name", namePrefix+".*"))
	return v, mc.Observe(err)
}


// ListInstanceGroups lists all InstanceGroups in the project and
// zone.
func (g *Cloud) ListInstanceGroups(zone string) ([]*compute.InstanceGroup, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newInstanceGroupMetricContext("list", zone)
	v, err := g.c.InstanceGroups().List(ctx, zone, filter.None)
	return v, mc.Observe(err)
}


// ListInstancesInInstanceGroup lists all the instances in a given
// instance group and state.
func (g *Cloud) ListInstancesInInstanceGroup(name string, zone string, state string) ([]*compute.InstanceWithNamedPorts, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newInstanceGroupMetricContext("list_instances", zone)
	req := &compute.InstanceGroupsListInstancesRequest{InstanceState: state}
	v, err := g.c.InstanceGroups().ListInstances(ctx, meta.ZonalKey(name, zone), req, filter.None)
	return v, mc.Observe(err)
}


// AddInstancesToInstanceGroup adds the given instances to the given
// instance group.
func (g *Cloud) AddInstancesToInstanceGroup(name string, zone string, instanceRefs []*compute.InstanceReference) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newInstanceGroupMetricContext("add_instances", zone)
	// TODO: should cull operation above this layer.
	if len(instanceRefs) == 0 {
		return nil
	}
	req := &compute.InstanceGroupsAddInstancesRequest{
		Instances: instanceRefs,
	}
	return mc.Observe(g.c.InstanceGroups().AddInstances(ctx, meta.ZonalKey(name, zone), req))
}


// RemoveInstancesFromInstanceGroup removes the given instances from
// the instance group.
func (g *Cloud) RemoveInstancesFromInstanceGroup(name string, zone string, instanceRefs []*compute.InstanceReference) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newInstanceGroupMetricContext("remove_instances", zone)
	// TODO: should cull operation above this layer.
	if len(instanceRefs) == 0 {
		return nil
	}
	req := &compute.InstanceGroupsRemoveInstancesRequest{
		Instances: instanceRefs,
	}
	return mc.Observe(g.c.InstanceGroups().RemoveInstances(ctx, meta.ZonalKey(name, zone), req))
}


// SetNamedPortsOfInstanceGroup sets the list of named ports on a given instance group
func (g *Cloud) SetNamedPortsOfInstanceGroup(igName, zone string, namedPorts []*compute.NamedPort) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newInstanceGroupMetricContext("set_namedports", zone)
	req := &compute.InstanceGroupsSetNamedPortsRequest{NamedPorts: namedPorts}
	return mc.Observe(g.c.InstanceGroups().SetNamedPorts(ctx, meta.ZonalKey(igName, zone), req))
}


// GetInstanceGroup returns an instance group by name.
func (g *Cloud) GetInstanceGroup(name string, zone string) (*compute.InstanceGroup, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newInstanceGroupMetricContext("get", zone)
	v, err := g.c.InstanceGroups().Get(ctx, meta.ZonalKey(name, zone))
	return v, mc.Observe(err)
}


func newInstancesMetricContext(request, zone string) *metricContext {
	return newGenericMetricContext("instances", request, unusedMetricLabel, zone, computeV1Version)
}


func splitNodesByZone(nodes []*v1.Node) map[string][]*v1.Node {
	zones := make(map[string][]*v1.Node)
	for _, n := range nodes {
		z := getZone(n)
		if z != defaultZone {
			zones[z] = append(zones[z], n)
		}
	}
	return zones
}


func getZone(n *v1.Node) string

func getZone(n *v1.Node) string {
	zone, ok := n.Labels[v1.LabelFailureDomainBetaZone]
	if !ok {
		return defaultZone
	}
	return zone
}


func makeHostURL(projectsAPIEndpoint, projectID, zone, host string) string {
	host = canonicalizeInstanceName(host)
	return projectsAPIEndpoint + strings.Join([]string{projectID, "zones", zone, "instances", host}, "/")
}


// ToInstanceReferences returns instance references by links
func (g *Cloud) ToInstanceReferences(zone string, instanceNames []string) (refs []*compute.InstanceReference) {
	for _, ins := range instanceNames {
		instanceLink := makeHostURL(g.projectsBasePath, g.projectID, zone, ins)
		refs = append(refs, &compute.InstanceReference{Instance: instanceLink})
	}
	return refs
}


// NodeAddresses is an implementation of Instances.NodeAddresses.
func (g *Cloud) NodeAddresses(ctx context.Context, nodeName types.NodeName) ([]v1.NodeAddress, error) {
	timeoutCtx, cancel := context.WithTimeout(ctx, 1*time.Hour)
	defer cancel()

	instanceName := string(nodeName)

	if g.useMetadataServer {
		// Use metadata server if possible
		if g.isCurrentInstance(instanceName) {

			nics, err := metadata.Get("instance/network-interfaces/")
			if err != nil {
				return nil, fmt.Errorf("couldn't get network interfaces: %v", err)
			}

			nicsArr := strings.Split(nics, "/\n")
			nodeAddresses := []v1.NodeAddress{}

			for _, nic := range nicsArr {

				if nic == "" {
					continue
				}

				internalIP, err := metadata.Get(fmt.Sprintf(networkInterfaceIP, nic))
				if err != nil {
					return nil, fmt.Errorf("couldn't get internal IP: %v", err)
				}
				nodeAddresses = append(nodeAddresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: internalIP})

				acs, err := metadata.Get(fmt.Sprintf(networkInterfaceAccessConfigs, nic))
				if err != nil {
					return nil, fmt.Errorf("couldn't get access configs: %v", err)
				}

				acsArr := strings.Split(acs, "/\n")

				for _, ac := range acsArr {

					if ac == "" {
						continue
					}

					externalIP, err := metadata.Get(fmt.Sprintf(networkInterfaceExternalIP, nic, ac))
					if err != nil {
						return nil, fmt.Errorf("couldn't get external IP: %v", err)
					}

					if externalIP != "" {
						nodeAddresses = append(nodeAddresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: externalIP})
					}
				}
			}

			internalDNSFull, err := metadata.Get("instance/hostname")
			if err != nil {
				klog.Warningf("couldn't get full internal DNS name: %v", err)
			} else {
				nodeAddresses = append(nodeAddresses,
					v1.NodeAddress{Type: v1.NodeInternalDNS, Address: internalDNSFull},
					v1.NodeAddress{Type: v1.NodeHostName, Address: internalDNSFull},
				)
			}
			return nodeAddresses, nil
		}
	}

	// Use GCE API
	instanceObj, err := g.getInstanceByName(instanceName)
	if err != nil {
		return nil, fmt.Errorf("couldn't get instance details: %v", err)
	}

	instance, err := g.c.Instances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(instanceObj.Name), instanceObj.Zone))
	if err != nil {
		return []v1.NodeAddress{}, fmt.Errorf("error while querying for instance: %v", err)
	}

	return nodeAddressesFromInstance(instance)
}


// NodeAddressesByProviderID will not be called from the node that is requesting this ID.
// i.e. metadata service and other local methods cannot be used here
func (g *Cloud) NodeAddressesByProviderID(ctx context.Context, providerID string) ([]v1.NodeAddress, error) {
	timeoutCtx, cancel := context.WithTimeout(ctx, 1*time.Hour)
	defer cancel()

	_, zone, name, err := splitProviderID(providerID)
	if err != nil {
		return []v1.NodeAddress{}, err
	}

	instance, err := g.c.Instances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
	if err != nil {
		return []v1.NodeAddress{}, fmt.Errorf("error while querying for providerID %q: %v", providerID, err)
	}

	return nodeAddressesFromInstance(instance)
}


// instanceByProviderID returns the cloudprovider instance of the node
// with the specified unique providerID
func (g *Cloud) instanceByProviderID(providerID string) (*gceInstance, error) {
	project, zone, name, err := splitProviderID(providerID)
	if err != nil {
		return nil, err
	}

	instance, err := g.getInstanceFromProjectInZoneByName(project, zone, name)
	if err != nil {
		if isHTTPErrorCode(err, http.StatusNotFound) {
			return nil, cloudprovider.InstanceNotFound
		}
		return nil, err
	}

	return instance, nil
}


// InstanceShutdownByProviderID returns true if the instance is in safe state to detach volumes
func (g *Cloud) InstanceShutdownByProviderID(ctx context.Context, providerID string) (bool, error) {
	return false, cloudprovider.NotImplemented
}


// InstanceShutdown returns true if the instance is in safe state to detach volumes
func (g *Cloud) InstanceShutdown(ctx context.Context, node *v1.Node) (bool, error) {
	return false, cloudprovider.NotImplemented
}


func nodeAddressesFromInstance(instance *compute.Instance) ([]v1.NodeAddress, error) {
	if len(instance.NetworkInterfaces) < 1 {
		return nil, fmt.Errorf("could not find network interfaces for instanceID %q", instance.Id)
	}
	nodeAddresses := []v1.NodeAddress{}

	for _, nic := range instance.NetworkInterfaces {
		nodeAddresses = append(nodeAddresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: nic.NetworkIP})
		for _, config := range nic.AccessConfigs {
			nodeAddresses = append(nodeAddresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: config.NatIP})
		}
	}

	return nodeAddresses, nil
}


// InstanceTypeByProviderID returns the cloudprovider instance type of the node
// with the specified unique providerID This method will not be called from the
// node that is requesting this ID. i.e. metadata service and other local
// methods cannot be used here
func (g *Cloud) InstanceTypeByProviderID(ctx context.Context, providerID string) (string, error) {
	instance, err := g.instanceByProviderID(providerID)
	if err != nil {
		return "", err
	}

	return instance.Type, nil
}


// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running.
// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager.
func (g *Cloud) InstanceExistsByProviderID(ctx context.Context, providerID string) (bool, error) {
	_, err := g.instanceByProviderID(providerID)
	if err != nil {
		if err == cloudprovider.InstanceNotFound {
			return false, nil
		}
		return false, err
	}

	return true, nil
}


// InstanceExists returns true if the instance with the given provider id still exists and is running.
// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager.
func (g *Cloud) InstanceExists(ctx context.Context, node *v1.Node) (bool, error) {
	providerID := node.Spec.ProviderID
	if providerID == "" {
		var err error
		if providerID, err = cloudprovider.GetInstanceProviderID(ctx, g, types.NodeName(node.Name)); err != nil {
			if err == cloudprovider.InstanceNotFound {
				return false, nil
			}
			return false, err
		}
	}
	return g.InstanceExistsByProviderID(ctx, providerID)
}


// InstanceMetadata returns metadata of the specified instance.
func (g *Cloud) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloudprovider.InstanceMetadata, error) {
	timeoutCtx, cancel := context.WithTimeout(ctx, 1*time.Hour)
	defer cancel()

	providerID := node.Spec.ProviderID
	if providerID == "" {
		var err error
		if providerID, err = cloudprovider.GetInstanceProviderID(ctx, g, types.NodeName(node.Name)); err != nil {
			return nil, err
		}
	}

	_, zone, name, err := splitProviderID(providerID)
	if err != nil {
		return nil, err
	}

	region, err := GetGCERegion(zone)
	if err != nil {
		return nil, err
	}

	instance, err := g.c.Instances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
	if err != nil {
		return nil, fmt.Errorf("error while querying for providerID %q: %v", providerID, err)
	}

	addresses, err := nodeAddressesFromInstance(instance)
	if err != nil {
		return nil, err
	}

	return &cloudprovider.InstanceMetadata{
		ProviderID:    providerID,
		InstanceType:  lastComponent(instance.MachineType),
		NodeAddresses: addresses,
		Zone:          zone,
		Region:        region,
	}, nil
}


// InstanceID returns the cloud provider ID of the node with the specified NodeName.
func (g *Cloud) InstanceID(ctx context.Context, nodeName types.NodeName) (string, error) {
	instanceName := mapNodeNameToInstanceName(nodeName)
	if g.useMetadataServer {
		// Use metadata, if possible, to fetch ID. See issue #12000
		if g.isCurrentInstance(instanceName) {
			projectID, zone, err := getProjectAndZone()
			if err == nil {
				return projectID + "/" + zone + "/" + canonicalizeInstanceName(instanceName), nil
			}
		}
	}
	instance, err := g.getInstanceByName(instanceName)
	if err != nil {
		return "", err
	}
	return g.projectID + "/" + instance.Zone + "/" + instance.Name, nil
}


// InstanceType returns the type of the specified node with the specified NodeName.
func (g *Cloud) InstanceType(ctx context.Context, nodeName types.NodeName) (string, error) {
	instanceName := mapNodeNameToInstanceName(nodeName)
	if g.useMetadataServer {
		// Use metadata, if possible, to fetch ID. See issue #12000
		if g.isCurrentInstance(instanceName) {
			mType, err := getCurrentMachineTypeViaMetadata()
			if err == nil {
				return mType, nil
			}
		}
	}
	instance, err := g.getInstanceByName(instanceName)
	if err != nil {
		return "", err
	}
	return instance.Type, nil
}


// AddSSHKeyToAllInstances adds an SSH public key as a legal identity for all instances
// expected format for the key is standard ssh-keygen format: <protocol> <blob>
func (g *Cloud) AddSSHKeyToAllInstances(ctx context.Context, user string, keyData []byte) error {
	ctx, cancel := context.WithTimeout(ctx, 1*time.Hour)
	defer cancel()

	return wait.Poll(2*time.Second, 30*time.Second, func() (bool, error) {
		project, err := g.c.Projects().Get(ctx, g.projectID)
		if err != nil {
			klog.Errorf("Could not get project: %v", err)
			return false, nil
		}
		keyString := fmt.Sprintf("%s:%s %s@%s", user, strings.TrimSpace(string(keyData)), user, user)
		found := false
		for _, item := range project.CommonInstanceMetadata.Items {
			if item.Key == "sshKeys" {
				if strings.Contains(*item.Value, keyString) {
					// We've already added the key
					klog.Info("SSHKey already in project metadata")
					return true, nil
				}
				value := *item.Value + "\n" + keyString
				item.Value = &value
				found = true
				break
			}
		}
		if !found {
			// This is super unlikely, so log.
			klog.Infof("Failed to find sshKeys metadata, creating a new item")
			project.CommonInstanceMetadata.Items = append(project.CommonInstanceMetadata.Items,
				&compute.MetadataItems{
					Key:   "sshKeys",
					Value: &keyString,
				})
		}

		mc := newInstancesMetricContext("add_ssh_key", "")
		err = g.c.Projects().SetCommonInstanceMetadata(ctx, g.projectID, project.CommonInstanceMetadata)
		mc.Observe(err)

		if err != nil {
			klog.Errorf("Could not Set Metadata: %v", err)
			return false, nil
		}
		klog.Infof("Successfully added sshKey to project metadata")
		return true, nil
	})
}


// GetAllCurrentZones returns all the zones in which k8s nodes are currently running
func (g *Cloud) GetAllCurrentZones() (sets.String, error) {
	if g.nodeInformerSynced == nil {
		klog.Warning("Cloud object does not have informers set, should only happen in E2E binary.")
		return g.GetAllZonesFromCloudProvider()
	}
	g.nodeZonesLock.Lock()
	defer g.nodeZonesLock.Unlock()
	if !g.nodeInformerSynced() {
		return nil, fmt.Errorf("node informer is not synced when trying to GetAllCurrentZones")
	}
	zones := sets.NewString()
	for zone, nodes := range g.nodeZones {
		if len(nodes) > 0 {
			zones.Insert(zone)
		}
	}
	return zones, nil
}


// GetAllZonesFromCloudProvider returns all the zones in which nodes are running
// Only use this in E2E tests to get zones, on real clusters this will
// get all zones with compute instances in them even if not k8s instances!!!
// ex. I have k8s nodes in us-central1-c and us-central1-b. I also have
// a non-k8s compute in us-central1-a. This func will return a,b, and c.
//
// TODO: this should be removed from the cloud provider.
func (g *Cloud) GetAllZonesFromCloudProvider() (sets.String, error) {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	zones := sets.NewString()
	for _, zone := range g.managedZones {
		instances, err := g.c.Instances().List(ctx, zone, filter.None)
		if err != nil {
			return sets.NewString(), err
		}
		if len(instances) > 0 {
			zones.Insert(zone)
		}
	}
	return zones, nil
}


// InsertInstance creates a new instance on GCP
func (g *Cloud) InsertInstance(project string, zone string, i *compute.Instance) error {
	ctx, cancel := cloud.ContextWithCallTimeout()
	defer cancel()

	mc := newInstancesMetricContext("create", zone)
	return mc.Observe(g.c.Instances().Insert(ctx, meta.ZonalKey(i.Name, zone), i))
}