// DisksAreAttached returns if disks are attached to the VM using controllers supported by the plugin.
// 1. Converts volPaths into canonical form so that it can be compared with the VM device path.
// 2. Segregates nodes by vCenter and Datacenter they are present in. This reduces calls to VC.
// 3. Creates go routines per VC-DC to find whether disks are attached to the nodes.
// 4. If the some of the VMs are not found or migrated then they are added to a list.
// 5. After successful execution of goroutines,
// 5a. If there are any VMs which needs to be retried, they are rediscovered and the whole operation is initiated again for only rediscovered VMs.
// 5b. If VMs are removed from vSphere inventory they are ignored.
func (vs *VSphere) DisksAreAttached(nodeVolumes map[k8stypes.NodeName][]string) (map[k8stypes.NodeName]map[string]bool, error) {
	disksAreAttachedInternal := func(nodeVolumes map[k8stypes.NodeName][]string) (map[k8stypes.NodeName]map[string]bool, error) {
		vs.vsphereVolumeMap.StartDiskVerification()

		// disksAreAttach checks whether disks are attached to the nodes.
		// Returns nodes that need to be retried if retry is true
		// Segregates nodes per VC and DC
		// Creates go routines per VC-DC to find whether disks are attached to the nodes.
		disksAreAttach := func(ctx context.Context, nodeVolumes map[k8stypes.NodeName][]string, attached map[string]map[string]bool, retry bool) ([]k8stypes.NodeName, error) {

			var wg sync.WaitGroup
			var localAttachedMaps []map[string]map[string]bool
			var nodesToRetry []k8stypes.NodeName
			var globalErr error
			globalErr = nil
			globalErrMutex := &sync.Mutex{}
			nodesToRetryMutex := &sync.Mutex{}

			// Segregate nodes according to VC-DC
			dcNodes := make(map[string][]k8stypes.NodeName)
			for nodeName := range nodeVolumes {
				nodeInfo, err := vs.nodeManager.GetNodeInfo(nodeName)
				if err != nil {
					klog.Errorf("Failed to get node info: %+v. err: %+v", nodeInfo.vm, err)
					return nodesToRetry, err
				}
				VC_DC := nodeInfo.vcServer + nodeInfo.dataCenter.String()
				dcNodes[VC_DC] = append(dcNodes[VC_DC], nodeName)
			}

			for _, nodeNames := range dcNodes {
				localAttachedMap := make(map[string]map[string]bool)
				localAttachedMaps = append(localAttachedMaps, localAttachedMap)
				// Start go routines per VC-DC to check disks are attached
				go func(nodes []k8stypes.NodeName) {
					nodesToRetryLocal, err := vs.checkDiskAttached(ctx, nodes, nodeVolumes, localAttachedMap, retry)
					if err != nil {
						if !vclib.IsManagedObjectNotFoundError(err) {
							globalErrMutex.Lock()
							globalErr = err
							globalErrMutex.Unlock()
							klog.Errorf("Failed to check disk attached for nodes: %+v. err: %+v", nodes, err)
						}
					}
					nodesToRetryMutex.Lock()
					nodesToRetry = append(nodesToRetry, nodesToRetryLocal...)
					nodesToRetryMutex.Unlock()
					wg.Done()
				}(nodeNames)
				wg.Add(1)
			}
			wg.Wait()
			if globalErr != nil {
				return nodesToRetry, globalErr
			}
			for _, localAttachedMap := range localAttachedMaps {
				for key, value := range localAttachedMap {
					attached[key] = value
				}
			}

			return nodesToRetry, nil
		}

		klog.V(4).Infof("Starting DisksAreAttached API for vSphere with nodeVolumes: %+v", nodeVolumes)
		// Create context
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()

		disksAttached := make(map[k8stypes.NodeName]map[string]bool)
		if len(nodeVolumes) == 0 {
			return disksAttached, nil
		}

		// Convert VolPaths into canonical form so that it can be compared with the VM device path.
		vmVolumes, err := vs.convertVolPathsToDevicePaths(ctx, nodeVolumes)
		if err != nil {
			klog.Errorf("Failed to convert volPaths to devicePaths: %+v. err: %+v", nodeVolumes, err)
			return nil, err
		}
		attached := make(map[string]map[string]bool)
		nodesToRetry, err := disksAreAttach(ctx, vmVolumes, attached, false)
		if err != nil {
			return nil, err
		}

		if len(nodesToRetry) != 0 {
			// Rediscover nodes which are need to be retried
			remainingNodesVolumes := make(map[k8stypes.NodeName][]string)
			for _, nodeName := range nodesToRetry {
				err = vs.nodeManager.RediscoverNode(nodeName)
				if err != nil {
					if err == vclib.ErrNoVMFound {
						klog.V(4).Infof("node %s not found. err: %+v", nodeName, err)
						continue
					}
					klog.Errorf("Failed to rediscover node %s. err: %+v", nodeName, err)
					return nil, err
				}
				remainingNodesVolumes[nodeName] = nodeVolumes[nodeName]
			}

			// If some remaining nodes are still registered
			if len(remainingNodesVolumes) != 0 {
				nodesToRetry, err = disksAreAttach(ctx, remainingNodesVolumes, attached, true)
				if err != nil || len(nodesToRetry) != 0 {
					klog.Errorf("Failed to retry disksAreAttach  for nodes %+v. err: %+v", remainingNodesVolumes, err)
					return nil, err
				}
			}

			for nodeName, volPaths := range attached {
				disksAttached[convertToK8sType(nodeName)] = volPaths
			}

		}
		klog.V(4).Infof("DisksAreAttached successfully executed. result: %+v", attached)
		// There could be nodes in cluster which do not have any pods with vsphere volumes running on them
		// such nodes won't be part of nodeVolumes map because attach-detach controller does not keep track
		// such nodes. But such nodes may still have dangling volumes on them and hence we need to scan all the
		// remaining nodes which weren't scanned by code previously.
		vs.BuildMissingVolumeNodeMap(ctx)
		// any volume which we could not verify will be removed from the map.
		vs.vsphereVolumeMap.RemoveUnverified()
		klog.V(4).Infof("current node volume map is: %+v", vs.vsphereVolumeMap.volumeNodeMap)
		return disksAttached, nil
	}
	requestTime := time.Now()
	attached, err := disksAreAttachedInternal(nodeVolumes)
	vclib.RecordvSphereMetric(vclib.OperationDisksAreAttached, requestTime, err)
	return attached, err
}


// CreateVolume creates a volume of given size (in KiB) and return the volume path.
// If the volumeOptions.Datastore is part of datastore cluster for example - [DatastoreCluster/sharedVmfs-0] then
// return value will be [DatastoreCluster/sharedVmfs-0] kubevols/<volume-name>.vmdk
// else return value will be [sharedVmfs-0] kubevols/<volume-name>.vmdk
func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVolumePath string, err error) {
	klog.V(1).Infof("Starting to create a vSphere volume with volumeOptions: %+v", volumeOptions)
	createVolumeInternal := func(volumeOptions *vclib.VolumeOptions) (canonicalVolumePath string, err error) {
		var datastoreInfo *vclib.DatastoreInfo
		var dsList []*vclib.DatastoreInfo

		// Create context
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx)
		if err != nil {
			return "", err
		}
		// If datastore not specified, then use default datastore
		datastoreName := strings.TrimSpace(volumeOptions.Datastore)
		if datastoreName == "" {
			datastoreName = strings.TrimSpace(vs.cfg.Workspace.DefaultDatastore)
		}
		// The given datastoreName may be present in more than one datacenter
		candidateDatastoreInfos, err := vs.FindDatastoreByName(ctx, datastoreName)
		if err != nil {
			return "", err
		}
		// Each of the datastores found is a candidate for Volume creation.
		// One of these will be selected based on given policy and/or zone.
		candidateDatastores := make(map[string]*vclib.DatastoreInfo)
		for _, dsInfo := range candidateDatastoreInfos {
			candidateDatastores[dsInfo.Info.Url] = dsInfo
		}

		var vmOptions *vclib.VMOptions
		var zonesToSearch []string

		if volumeOptions.SelectedNode != nil {
			if len(volumeOptions.Zone) > 1 {
				// In waitForFirstConsumer mode, if more than one allowedTopologies is specified, the volume should satisfy all these.
				zonesToSearch = volumeOptions.Zone
			} else {
				// Pick the selectedNode's zone, if available.
				nodeInfo, err := vs.nodeManager.GetNodeInfoWithNodeObject(volumeOptions.SelectedNode)
				if err != nil {
					klog.Errorf("Unable to get node information for %s. err: %+v", volumeOptions.SelectedNode.Name, err)
					return "", err
				}
				klog.V(4).Infof("selectedNode info : %s", nodeInfo)
				if nodeInfo.zone != nil && nodeInfo.zone.FailureDomain != "" {
					zonesToSearch = append(zonesToSearch, nodeInfo.zone.FailureDomain)
				}
			}
		} else {
			// If no selectedNode, pick allowedTopologies, if provided.
			zonesToSearch = volumeOptions.Zone
		}
		klog.V(1).Infof("Volume topology : %s", zonesToSearch)

		if volumeOptions.VSANStorageProfileData != "" || volumeOptions.StoragePolicyName != "" {
			// If datastore and zone are specified, first validate if the datastore is in the provided zone.
			if len(zonesToSearch) != 0 && volumeOptions.Datastore != "" {
				klog.V(4).Infof("Specified zone : %s, datastore : %s", zonesToSearch, volumeOptions.Datastore)
				dsList, err = getDatastoresForZone(ctx, vs.nodeManager, zonesToSearch)
				if err != nil {
					klog.Errorf("Failed to find a shared datastore matching zone %s. err: %+v", zonesToSearch, err)
					return "", err
				}

				// Validate if the datastore provided belongs to the zone. If not, fail the operation.
				found := false
				for _, ds := range dsList {
					if datastoreInfo, found = candidateDatastores[ds.Info.Url]; found {
						break
					}
				}
				if !found {
					err := fmt.Errorf("The specified datastore %s does not match the provided zones : %s", volumeOptions.Datastore, zonesToSearch)
					klog.Error(err)
					return "", err
				}
			}
			// Acquire a read lock to ensure multiple PVC requests can be processed simultaneously.
			cleanUpDummyVMLock.RLock()
			defer cleanUpDummyVMLock.RUnlock()
			// Create a new background routine that will delete any dummy VM's that are left stale.
			// This routine will get executed for every 5 minutes and gets initiated only once in its entire lifetime.
			cleanUpRoutineInitLock.Lock()
			if !cleanUpRoutineInitialized {
				klog.V(1).Infof("Starting a clean up routine to remove stale dummy VM's")
				go vs.cleanUpDummyVMs(DummyVMPrefixName)
				cleanUpRoutineInitialized = true
			}
			cleanUpRoutineInitLock.Unlock()
		}
		if volumeOptions.StoragePolicyName != "" && volumeOptions.Datastore == "" {
			if len(zonesToSearch) == 0 {
				klog.V(4).Infof("Selecting a shared datastore as per the storage policy %s", volumeOptions.StoragePolicyName)
				datastoreInfo, err = getPbmCompatibleDatastore(ctx, vsi.conn.Client, volumeOptions.StoragePolicyName, vs.nodeManager)
			} else {
				// If zone is specified, first get the datastores in the zone.
				dsList, err = getDatastoresForZone(ctx, vs.nodeManager, zonesToSearch)

				if err != nil {
					klog.Errorf("Failed to find a shared datastore matching zone %s. err: %+v", zonesToSearch, err)
					return "", err
				}

				klog.V(4).Infof("Specified zone : %s. Picking a datastore as per the storage policy %s among the zoned datastores : %s", zonesToSearch,
					volumeOptions.StoragePolicyName, dsList)
				// Among the compatible datastores, select the one based on the maximum free space.
				datastoreInfo, err = getPbmCompatibleZonedDatastore(ctx, vsi.conn.Client, volumeOptions.StoragePolicyName, dsList)
			}
			if err != nil {
				klog.Errorf("Failed to get pbm compatible datastore with storagePolicy: %s. err: %+v", volumeOptions.StoragePolicyName, err)
				return "", err
			}
			klog.V(1).Infof("Datastore selected as per policy : %s", datastoreInfo.Info.Name)
		} else {
			// If zone is specified, pick the datastore in the zone with maximum free space within the zone.
			if volumeOptions.Datastore == "" && len(zonesToSearch) != 0 {
				klog.V(4).Infof("Specified zone : %s", zonesToSearch)
				dsList, err = getDatastoresForZone(ctx, vs.nodeManager, zonesToSearch)

				if err != nil {
					klog.Errorf("Failed to find a shared datastore matching zone %s. err: %+v", zonesToSearch, err)
					return "", err
				}
				// If unable to get any datastore, fail the operation
				if len(dsList) == 0 {
					err := fmt.Errorf("Failed to find a shared datastore matching zone %s", zonesToSearch)
					klog.Error(err)
					return "", err
				}

				datastoreInfo, err = getMostFreeDatastore(ctx, nil, dsList)
				if err != nil {
					klog.Errorf("Failed to get shared datastore: %+v", err)
					return "", err
				}
				klog.V(1).Infof("Specified zone : %s. Selected datastore : %s", zonesToSearch, datastoreInfo.Info.Name)
			} else {
				var sharedDsList []*vclib.DatastoreInfo
				var err error
				if len(zonesToSearch) == 0 {
					// If zone is not provided, get the shared datastore across all node VMs.
					klog.V(4).Infof("Validating if datastore %s is shared across all node VMs", datastoreName)
					sharedDSFinder := &sharedDatastore{
						nodeManager:         vs.nodeManager,
						candidateDatastores: candidateDatastoreInfos,
					}
					datastoreInfo, err = sharedDSFinder.getSharedDatastore(ctx)
					if err != nil {
						klog.Errorf("Failed to get shared datastore: %+v", err)
						return "", err
					}
					if datastoreInfo == nil {
						err = fmt.Errorf("The specified datastore %s is not a shared datastore across node VMs", datastoreName)
						klog.Error(err)
						return "", err
					}
				} else {
					// If zone is provided, get the shared datastores in that zone.
					klog.V(4).Infof("Validating if datastore %s is in zone %s ", datastoreName, zonesToSearch)
					sharedDsList, err = getDatastoresForZone(ctx, vs.nodeManager, zonesToSearch)
					if err != nil {
						klog.Errorf("Failed to find a shared datastore matching zone %s. err: %+v", zonesToSearch, err)
						return "", err
					}
					found := false
					for _, sharedDs := range sharedDsList {
						if datastoreInfo, found = candidateDatastores[sharedDs.Info.Url]; found {
							klog.V(4).Infof("Datastore validation succeeded")
							found = true
							break
						}
					}
					if !found {
						err = fmt.Errorf("The specified datastore %s does not match the provided zones : %s", datastoreName, zonesToSearch)
						klog.Error(err)
						return "", err
					}
				}
			}
		}

		// if datastoreInfo is still not determined, it is an error condition
		if datastoreInfo == nil {
			klog.Errorf("ambiguous datastore name %s, cannot be found among: %v", datastoreName, candidateDatastoreInfos)
			return "", fmt.Errorf("ambiguous datastore name %s", datastoreName)
		}
		ds := datastoreInfo.Datastore
		volumeOptions.Datastore = datastoreInfo.Info.Name
		vmOptions, err = vs.setVMOptions(ctx, vsi.conn, ds)
		if err != nil {
			klog.Errorf("failed to set VM options required to create a vsphere volume. err: %+v", err)
			return "", err
		}
		kubeVolsPath := filepath.Clean(ds.Path(VolDir)) + "/"
		err = ds.CreateDirectory(ctx, kubeVolsPath, false)
		if err != nil && err != vclib.ErrFileAlreadyExist {
			klog.Errorf("Cannot create dir %#v. err %s", kubeVolsPath, err)
			return "", err
		}
		volumePath := kubeVolsPath + volumeOptions.Name + ".vmdk"
		disk := diskmanagers.VirtualDisk{
			DiskPath:      volumePath,
			VolumeOptions: volumeOptions,
			VMOptions:     vmOptions,
		}
		volumePath, err = disk.Create(ctx, ds)
		if err != nil {
			klog.Errorf("Failed to create a vsphere volume with volumeOptions: %+v on datastore: %s. err: %+v", volumeOptions, ds, err)
			return "", err
		}
		// Get the canonical path for the volume path.
		canonicalVolumePath, err = getcanonicalVolumePath(ctx, datastoreInfo.Datacenter, volumePath)
		if err != nil {
			klog.Errorf("Failed to get canonical vsphere volume path for volume: %s with volumeOptions: %+v on datastore: %s. err: %+v", volumePath, volumeOptions, ds, err)
			return "", err
		}
		if filepath.Base(datastoreName) != datastoreName {
			// If datastore is within cluster, add cluster path to the volumePath
			canonicalVolumePath = strings.Replace(canonicalVolumePath, filepath.Base(datastoreName), datastoreName, 1)
		}
		return canonicalVolumePath, nil
	}
	requestTime := time.Now()
	canonicalVolumePath, err = createVolumeInternal(volumeOptions)
	vclib.RecordCreateVolumeMetric(volumeOptions, requestTime, err)
	klog.V(4).Infof("The canonical volume path for the newly created vSphere volume is %q", canonicalVolumePath)
	return canonicalVolumePath, err
}


// DeleteVolume deletes a volume given volume name.
func (vs *VSphere) DeleteVolume(vmDiskPath string) error {
	klog.V(1).Infof("Starting to delete vSphere volume with vmDiskPath: %s", vmDiskPath)
	deleteVolumeInternal := func(vmDiskPath string) error {
		// Create context
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx)
		if err != nil {
			return err
		}
		dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter)
		if err != nil {
			return err
		}
		disk := diskmanagers.VirtualDisk{
			DiskPath:      vmDiskPath,
			VolumeOptions: &vclib.VolumeOptions{},
			VMOptions:     &vclib.VMOptions{},
		}
		err = disk.Delete(ctx, dc)
		if err != nil {
			klog.Errorf("Failed to delete vsphere volume with vmDiskPath: %s. err: %+v", vmDiskPath, err)
		}
		return err
	}
	requestTime := time.Now()
	err := deleteVolumeInternal(vmDiskPath)
	vclib.RecordvSphereMetric(vclib.OperationDeleteVolume, requestTime, err)
	return err
}


func (vs *VSphere) HasClusterID() bool

// HasClusterID returns true if the cluster has a clusterID
func (vs *VSphere) HasClusterID() bool {
	return true
}


// Notification handler when node is added into k8s cluster.
func (vs *VSphere) NodeAdded(obj interface{}) {
	node, ok := obj.(*v1.Node)
	if node == nil || !ok {
		klog.Warningf("NodeAdded: unrecognized object %+v", obj)
		return
	}

	klog.V(4).Infof("Node added: %+v", node)
	if err := vs.nodeManager.RegisterNode(node); err != nil {
		klog.Errorf("failed to add node %+v: %v", node, err)
	}
	vs.setNodeZoneLabels(node)
}


// Node zone labels sync function, intended to be called periodically within kube-controller-manager.
func (vs *VSphere) syncNodeZoneLabels(_ interface{}, newObj interface{}) {
	node, ok := newObj.(*v1.Node)
	if node == nil || !ok {
		klog.Warningf("NodeUpdated: unrecognized object %+v", newObj)
		return
	}

	// Populate zone and region labels if needed.
	// This logic engages only if credentials provided via secret.
	// Returns early if topology labels are already presented.
	// https://github.com/kubernetes/kubernetes/issues/75175
	if vs.isSecretInfoProvided && vs.isZoneEnabled() {
		labels := node.GetLabels()
		_, zoneOk := labels[v1.LabelTopologyZone]
		_, regionOk := labels[v1.LabelTopologyRegion]
		if zoneOk && regionOk {
			klog.V(6).Infof("Node topology labels are already populated")
			return
		}
		klog.V(4).Infof("Topology labels was not found, trying to populate for node %s", node.Name)
		vs.setNodeZoneLabels(node)
	}
}


func (vs *VSphere) setNodeZoneLabels(node *v1.Node) {
	nodeZone := node.ObjectMeta.Labels[v1.LabelTopologyZone]
	nodeRegion := node.ObjectMeta.Labels[v1.LabelTopologyRegion]
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	if vs.isSecretInfoProvided && vs.isZoneEnabled() {
		zone, err := vs.GetZoneByProviderID(ctx, node.Spec.ProviderID)
		if err != nil {
			klog.Warningf("Can not get Zones from vCenter: %v", err)
		}

		if zone.FailureDomain != nodeZone || zone.Region != nodeRegion {
			updatedNode := node.DeepCopy()
			labels := updatedNode.ObjectMeta.Labels
			if labels == nil {
				labels = make(map[string]string)
			}
			labels[v1.LabelTopologyZone] = zone.FailureDomain
			labels[v1.LabelTopologyRegion] = zone.Region

			err = tryUpdateNode(ctx, vs.kubeClient, updatedNode)
			if err != nil {
				klog.Errorf("vSphere cloud provider can not update node with zones info: %v", err)
			} else {
				klog.V(4).Infof("Node %s updated with zone and region labels", updatedNode.Name)
			}
		}
	}
}


func tryUpdateNode(ctx context.Context, client clientset.Interface, updatedNode *v1.Node) error {
	for i := 0; i < updateNodeRetryCount; i++ {
		_, err := client.CoreV1().Nodes().Update(ctx, updatedNode, metav1.UpdateOptions{})
		if err != nil {
			if !apierrors.IsConflict(err) {
				return fmt.Errorf("vSphere cloud provider can not update node with zones info: %v", err)
			}
		} else {
			return nil
		}
	}
	return fmt.Errorf("update node exceeds retry count")
}


// Notification handler when node is removed from k8s cluster.
func (vs *VSphere) NodeDeleted(obj interface{}) {
	node, ok := obj.(*v1.Node)
	if node == nil || !ok {
		klog.Warningf("NodeDeleted: unrecognized object %+v", obj)
		return
	}

	klog.V(4).Infof("Node deleted: %+v", node)
	if err := vs.nodeManager.UnRegisterNode(node); err != nil {
		klog.Errorf("failed to delete node %s: %v", node.Name, err)
	}
}


// Notification handler when credentials secret is added.
func (vs *VSphere) SecretAdded(obj interface{}) {
	secret, ok := obj.(*v1.Secret)
	if secret == nil || !ok {
		klog.Warningf("Unrecognized secret object %T", obj)
		return
	}

	if secret.Name != vs.cfg.Global.SecretName ||
		secret.Namespace != vs.cfg.Global.SecretNamespace {
		return
	}

	klog.V(4).Infof("refreshing node cache for secret: %s/%s", secret.Namespace, secret.Name)
	vs.refreshNodesForSecretChange()
}


// Notification handler when credentials secret is updated.
func (vs *VSphere) SecretUpdated(obj interface{}, newObj interface{}) {
	oldSecret, ok := obj.(*v1.Secret)
	if oldSecret == nil || !ok {
		klog.Warningf("Unrecognized secret object %T", obj)
		return
	}

	secret, ok := newObj.(*v1.Secret)
	if secret == nil || !ok {
		klog.Warningf("Unrecognized secret object %T", newObj)
		return
	}

	if secret.Name != vs.cfg.Global.SecretName ||
		secret.Namespace != vs.cfg.Global.SecretNamespace ||
		reflect.DeepEqual(secret.Data, oldSecret.Data) {
		return
	}

	klog.V(4).Infof("refreshing node cache for secret: %s/%s", secret.Namespace, secret.Name)
	vs.refreshNodesForSecretChange()
}


func (vs *VSphere) refreshNodesForSecretChange() {
	err := vs.nodeManager.refreshNodes()
	if err != nil {
		klog.Errorf("failed to rediscover nodes: %v", err)
	}
}


func (vs *VSphere) NodeManager() (nodeManager *NodeManager) {
	if vs == nil {
		return nil
	}
	return vs.nodeManager
}


func withTagsClient(ctx context.Context, connection *vclib.VSphereConnection, f func(c *rest.Client) error) error {
	c := rest.NewClient(connection.Client)
	signer, err := connection.Signer(ctx, connection.Client)
	if err != nil {
		return err
	}
	if signer == nil {
		user := url.UserPassword(connection.Username, connection.Password)
		err = c.Login(ctx, user)
	} else {
		err = c.LoginByToken(c.WithSigner(ctx, signer))
	}
	if err != nil {
		return err
	}

	defer func() {
		if err := c.Logout(ctx); err != nil {
			klog.Errorf("failed to logout: %v", err)
		}
	}()
	return f(c)
}


func (vs *VSphere) getZoneByVmUUIDAndNodeName(ctx context.Context, vmUUID string, nodeName k8stypes.NodeName) (cloudprovider.Zone, error) {
	zone := cloudprovider.Zone{}

	vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx)
	if err != nil {
		klog.Errorf("Cannot connect to vsphere. Get zone for node %s error", nodeName)
		return cloudprovider.Zone{}, err
	}
	dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter)
	if err != nil {
		klog.Errorf("Cannot connect to datacenter. Get zone for node %s error", nodeName)
		return cloudprovider.Zone{}, err
	}
	vmHost, err := dc.GetHostByVMUUID(ctx, vmUUID)
	if err != nil {
		klog.Errorf("Cannot find VM runtime host. Get zone for node %s error", nodeName)
		return cloudprovider.Zone{}, err
	}

	pc := vsi.conn.Client.ServiceContent.PropertyCollector
	err = withTagsClient(ctx, vsi.conn, func(c *rest.Client) error {
		client := tags.NewManager(c)
		// example result: ["Folder", "Datacenter", "Cluster", "Host"]
		objects, err := mo.Ancestors(ctx, vsi.conn.Client, pc, *vmHost)
		if err != nil {
			return err
		}

		// search the hierarchy, example order: ["Host", "Cluster", "Datacenter", "Folder"]
		for i := range objects {
			obj := objects[len(objects)-1-i]
			attachedTags, err := client.ListAttachedTags(ctx, obj)
			if err != nil {
				klog.Errorf("Cannot list attached tags. Get zone for node %s: %s", nodeName, err)
				return err
			}
			for _, value := range attachedTags {
				tag, err := client.GetTag(ctx, value)
				if err != nil {
					klog.Errorf("Get tag %s: %s", value, err)
					return err
				}
				category, err := client.GetCategory(ctx, tag.CategoryID)
				if err != nil {
					klog.Errorf("Get category %s error", value)
					return err
				}

				found := func() {
					klog.Errorf("Found %q tag (%s) for %s attached to %s", category.Name, tag.Name, vmUUID, obj.Reference())
				}
				switch {
				case category.Name == vs.cfg.Labels.Zone:
					zone.FailureDomain = tag.Name
					found()
				case category.Name == vs.cfg.Labels.Region:
					zone.Region = tag.Name
					found()
				}

				if zone.FailureDomain != "" && zone.Region != "" {
					return nil
				}
			}
		}

		if zone.Region == "" {
			return fmt.Errorf("vSphere region category %q does not match any tags for node %s [%s]", vs.cfg.Labels.Region, nodeName, vmUUID)
		}
		if zone.FailureDomain == "" {
			return fmt.Errorf("vSphere zone category %q does not match any tags for node %s [%s]", vs.cfg.Labels.Zone, nodeName, vmUUID)
		}

		return nil
	})
	if err != nil {
		klog.Errorf("Get zone for node %s: %s", nodeName, err)
		return cloudprovider.Zone{}, err
	}
	return zone, nil
}


// GetZone implements Zones.GetZone
func (vs *VSphere) GetZone(ctx context.Context) (cloudprovider.Zone, error) {
	nodeName, err := vs.CurrentNodeName(ctx, vs.hostName)
	if err != nil {
		klog.Errorf("Cannot get node name.")
		return cloudprovider.Zone{}, err
	}
	return vs.getZoneByVmUUIDAndNodeName(ctx, vs.vmUUID, nodeName)
}


func (vs *VSphere) GetZoneByNodeName(ctx context.Context, nodeName k8stypes.NodeName) (cloudprovider.Zone, error) {
	return cloudprovider.Zone{}, cloudprovider.NotImplemented
}


func (vs *VSphere) GetZoneByProviderID(ctx context.Context, providerID string) (cloudprovider.Zone, error) {
	var nodeName k8stypes.NodeName
	vmUUID := strings.Replace(providerID, providerIDPrefix, "", 1)

	for nName, nInfo := range vs.nodeManager.nodeInfoMap {
		if nInfo.vmUUID == vmUUID {
			nodeName = convertToK8sType(nName)
			break
		}
	}

	return vs.getZoneByVmUUIDAndNodeName(ctx, vmUUID, nodeName)
}


// GetLabelsForVolume implements the PVLabeler interface for VSphere
// since this interface is used by the PV label admission controller.
func (vs *VSphere) GetLabelsForVolume(ctx context.Context, pv *v1.PersistentVolume) (map[string]string, error) {
	// ignore if zones not enabled
	if !vs.isZoneEnabled() {
		klog.V(4).Infof("Zone labels for volume is not enabled in vsphere.conf")
		return nil, nil
	}
	// ignore if not vSphere volume
	if pv.Spec.VsphereVolume == nil {
		return nil, nil
	}
	return vs.GetVolumeLabels(pv.Spec.VsphereVolume.VolumePath)
}


// GetVolumeLabels returns the well known zone and region labels for given volume
func (vs *VSphere) GetVolumeLabels(volumePath string) (map[string]string, error) {
	// Create context
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	// if zones is not enabled return no labels
	if !vs.isZoneEnabled() {
		klog.V(4).Infof("Volume zone labels is not enabled in vsphere.conf")
		return nil, nil
	}

	// Find the datastore on which this volume resides
	datastorePathObj, err := vclib.GetDatastorePathObjFromVMDiskPath(volumePath)
	if err != nil {
		klog.Errorf("Failed to get datastore for volume: %v: %+v", volumePath, err)
		return nil, err
	}
	dsInfos, err := vs.FindDatastoreByName(ctx, datastorePathObj.Datastore)
	if err != nil {
		klog.Errorf("Failed to get datastore by name: %v: %+v", datastorePathObj.Datastore, err)
		return nil, err
	}
	var datastore *vclib.Datastore
	for _, dsInfo := range dsInfos {
		if dsInfo.Datastore.Exists(ctx, datastorePathObj.Path) {
			datastore = dsInfo.Datastore
		}
	}
	if datastore == nil {
		klog.Errorf("Could not find %s among %v", volumePath, dsInfos)
		return nil, fmt.Errorf("could not find the datastore for volume: %s", volumePath)
	}

	dsZones, err := vs.GetZonesForDatastore(ctx, datastore)
	if err != nil {
		klog.Errorf("Failed to get zones for datastore %v: %+v", datastorePathObj.Datastore, err)
		return nil, err
	}
	dsZones, err = vs.collapseZonesInRegion(ctx, dsZones)
	if err != nil {
		klog.Errorf("Failed to collapse zones. %v", err)
		return nil, err
	}
	// FIXME: For now, pick the first zone of datastore as the zone of volume
	labels := make(map[string]string)
	if len(dsZones) > 0 {
		labels[v1.LabelTopologyRegion] = dsZones[0].Region
		labels[v1.LabelTopologyZone] = dsZones[0].FailureDomain
	}
	return labels, nil
}


// collapse all zones in same region. Join FailureDomain with well known separator
func (vs *VSphere) collapseZonesInRegion(ctx context.Context, zones []cloudprovider.Zone) ([]cloudprovider.Zone, error) {
	// first create a map of region -> list of zones in that region
	regionToZones := make(map[string][]string)
	for _, zone := range zones {
		fds, exists := regionToZones[zone.Region]
		if !exists {
			fds = make([]string, 0)
		}
		regionToZones[zone.Region] = append(fds, zone.FailureDomain)
	}

	// Join all fds in same region and return Zone instances
	collapsedZones := make([]cloudprovider.Zone, 0)
	for region, fds := range regionToZones {
		fdSet := sets.NewString(fds...)
		appendedZone := volumehelpers.ZonesSetToLabelValue(fdSet)
		collapsedZones = append(collapsedZones, cloudprovider.Zone{FailureDomain: appendedZone, Region: region})
	}
	return collapsedZones, nil
}


// GetZonesForDatastore returns all the zones from which this datastore is visible
func (vs *VSphere) GetZonesForDatastore(ctx context.Context, datastore *vclib.Datastore) ([]cloudprovider.Zone, error) {
	vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx)
	if err != nil {
		klog.Errorf("Failed to get vSphere instance: %+v", err)
		return nil, err
	}

	// get the hosts mounted on this datastore
	// datastore -> ["host-1", "host-2", "host-3", ...]
	dsHosts, err := datastore.GetDatastoreHostMounts(ctx)
	if err != nil {
		klog.Errorf("Failed to get datastore host mounts for %v: %+v", datastore, err)
		return nil, err
	}
	klog.V(4).Infof("Got host mounts for datastore: %v: %v", datastore, dsHosts)

	// compute map of zone to list of hosts in that zone across all hosts in vsphere
	// zone -> ["host-i", "host-j", "host-k", ...]
	zoneToHosts, err := vs.GetZoneToHosts(ctx, vsi)
	if err != nil {
		klog.Errorf("Failed to get zones for hosts: %+v", err)
		return nil, err
	}
	klog.V(4).Infof("Got zone to hosts: %v", zoneToHosts)

	// datastore belongs to a zone if all hosts in that zone mount that datastore
	dsZones := make([]cloudprovider.Zone, 0)
	for zone, zoneHosts := range zoneToHosts {
		// if zone is valid and zoneHosts is a subset of dsHosts, then add zone
		if zone.Region != "" && containsAll(dsHosts, zoneHosts) {
			dsZones = append(dsZones, zone)
		}
	}
	klog.V(4).Infof("Datastore %s belongs to zones: %v", datastore, dsZones)
	return dsZones, nil
}


// GetZoneToHosts returns a map of 'zone' -> 'list of hosts in that zone' in given VC
func (vs *VSphere) GetZoneToHosts(ctx context.Context, vsi *VSphereInstance) (map[cloudprovider.Zone][]vmwaretypes.ManagedObjectReference, error) {
	// Approach is to find tags with the category of 'vs.cfg.Labels.Zone'
	zoneToHosts := make(map[cloudprovider.Zone][]vmwaretypes.ManagedObjectReference)

	getHostsInTagCategory := func(ctx context.Context, tagCategoryName string) (map[vmwaretypes.ManagedObjectReference]string, error) {

		hostToTag := make(map[vmwaretypes.ManagedObjectReference]string)
		err := withTagsClient(ctx, vsi.conn, func(c *rest.Client) error {
			// Look whether the zone/region tag is defined in VC
			tagManager := tags.NewManager(c)
			tagsForCat, err := tagManager.GetTagsForCategory(ctx, tagCategoryName)
			if err != nil {
				klog.V(4).Infof("No tags with category %s exists in VC. So ignoring.", tagCategoryName)
				// return a special error so that tag unavailability can be ignored
				return ErrNoZoneTagInVC
			}
			klog.V(4).Infof("List of tags under category %s: %v", tagCategoryName, tagsForCat)

			// Each such tag is a different 'zone' marked in vCenter.
			// Query for objects associated with each tag. Consider Host, Cluster and Datacenter kind of objects.
			tagToObjects := make(map[string][]mo.Reference)
			for _, tag := range tagsForCat {
				klog.V(4).Infof("Getting objects associated with tag %s", tag.Name)
				objects, err := tagManager.ListAttachedObjects(ctx, tag.Name)
				if err != nil {
					klog.Errorf("Error fetching objects associated with zone tag %s: %+v", tag.Name, err)
					return err
				}
				tagToObjects[tag.Name] = objects
			}
			klog.V(4).Infof("Map of tag to objects: %v", tagToObjects)

			// Infer zone for hosts within Datacenter, hosts within clusters and hosts - in this order of increasing priority
			// The below nested for-loops goes over all the objects in tagToObjects three times over.
			for _, moType := range []string{vclib.DatacenterType, vclib.ClusterComputeResourceType, vclib.HostSystemType} {
				for tagName, objects := range tagToObjects {
					for _, obj := range objects {
						if obj.Reference().Type == moType {
							klog.V(4).Infof("Found zone tag %s associated with %s of type %T: %s", tagName, obj, obj, obj.Reference().Value)
							switch moType {
							case "Datacenter":
								// mark that all hosts in this datacenter has tag applied
								dcObjRef := object.NewReference(vsi.conn.Client, obj.Reference())
								klog.V(4).Infof("Converted mo obj %v to govmomi object ref %v", obj, dcObjRef)
								dcObj, ok := dcObjRef.(*object.Datacenter)
								if !ok {
									errMsg := fmt.Sprintf("Not able to convert object to Datacenter %v", obj)
									klog.Errorf(errMsg)
									return errors.New(errMsg)
								}
								klog.V(4).Infof("Converted to object Datacenter %v", dcObj)
								dc := vclib.Datacenter{Datacenter: dcObj}
								hosts, err := dc.GetAllHosts(ctx)
								if err != nil {
									klog.Errorf("Could not get hosts from datacenter %v: %+v", dc, err)
									return err
								}
								for _, host := range hosts {
									hostToTag[host] = tagName
								}
							case "ClusterComputeResource":
								// mark that all hosts in this cluster has tag applied
								clusterObjRef := object.NewReference(vsi.conn.Client, obj.Reference())
								clusterObj, ok := clusterObjRef.(*object.ClusterComputeResource)
								if !ok {
									errMsg := fmt.Sprintf("Not able to convert object ClusterComputeResource %v", obj)
									klog.Errorf(errMsg)
									return errors.New(errMsg)
								}
								hostSystemList, err := clusterObj.Hosts(ctx)
								if err != nil {
									klog.Errorf("Not able to get hosts in cluster %v: %+v", clusterObj, err)
									return err
								}
								for _, host := range hostSystemList {
									hostToTag[host.Reference()] = tagName
								}
							case "HostSystem":
								// mark that this host has tag applied
								hostToTag[obj.Reference()] = tagName
							}
						}
					}
				}
			}
			return nil // no error
		})
		if err != nil {
			klog.Errorf("Error processing tag category %s: %+v", tagCategoryName, err)
			return nil, err
		}
		klog.V(6).Infof("Computed hostToTag: %v", hostToTag)
		return hostToTag, nil
	}

	hostToZone, err := getHostsInTagCategory(ctx, vs.cfg.Labels.Zone)
	if err != nil {
		if err == ErrNoZoneTagInVC {
			return zoneToHosts, nil
		}
		klog.Errorf("Get hosts in tag category %s failed: %+v", vs.cfg.Labels.Zone, err)
		return nil, err
	}

	hostToRegion, err := getHostsInTagCategory(ctx, vs.cfg.Labels.Region)
	if err != nil {
		if err == ErrNoZoneTagInVC {
			return zoneToHosts, nil
		}
		klog.Errorf("Get hosts in tag category %s failed: %+v", vs.cfg.Labels.Region, err)
		return nil, err
	}

	// populate zoneToHosts based on hostToZone and hostToRegion
	klog.V(6).Infof("hostToZone: %v", hostToZone)
	klog.V(6).Infof("hostToRegion: %v", hostToRegion)
	for host, zone := range hostToZone {
		region, regionExists := hostToRegion[host]
		if !regionExists {
			klog.Errorf("Host %s has a zone, but no region. So ignoring.", host)
			continue
		}
		cpZone := cloudprovider.Zone{FailureDomain: zone, Region: region}
		hosts, exists := zoneToHosts[cpZone]
		if !exists {
			hosts = make([]vmwaretypes.ManagedObjectReference, 0)
		}
		zoneToHosts[cpZone] = append(hosts, host)
	}
	klog.V(4).Infof("Final zoneToHosts: %v", zoneToHosts)
	return zoneToHosts, nil
}


// returns true if s1 contains all elements from s2; false otherwise
func containsAll(s1 []vmwaretypes.ManagedObjectReference, s2 []vmwaretypes.ManagedObjectReference) bool {
	// put all elements of s1 into a map
	s1Map := make(map[vmwaretypes.ManagedObjectReference]bool)
	for _, mor := range s1 {
		s1Map[mor] = true
	}
	// verify if all elements of s2 are present in s1Map
	for _, mor := range s2 {
		if _, found := s1Map[mor]; !found {
			return false
		}
	}
	return true
}


func GetVSphere() (*VSphere, error)

// GetVSphere reads vSphere configuration from system environment and construct vSphere object
func GetVSphere() (*VSphere, error) {
	cfg, err := getVSphereConfig()
	if err != nil {
		return nil, err
	}
	vs, err := newControllerNode(*cfg)
	if err != nil {
		return nil, err
	}
	return vs, nil
}


func getVSphereConfig() (*VSphereConfig, error) {
	confFileLocation := os.Getenv(vSphereConfFileEnvVar)
	if confFileLocation == "" {
		return nil, fmt.Errorf("Env variable 'VSPHERE_CONF_FILE' is not set.")
	}
	confFile, err := os.Open(confFileLocation)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := confFile.Close(); err != nil {
			klog.Errorf("failed to close config file: %v", err)
		}
	}()

	cfg, err := readConfig(confFile)
	if err != nil {
		return nil, err
	}
	return &cfg, nil
}


// Returns the accessible datastores for the given node VM.
func getAccessibleDatastores(ctx context.Context, nodeVmDetail *NodeDetails, nodeManager *NodeManager) ([]*vclib.DatastoreInfo, error) {
	accessibleDatastores, err := nodeVmDetail.vm.GetAllAccessibleDatastores(ctx)
	if err != nil {
		// Check if the node VM is not found which indicates that the node info in the node manager is stale.
		// If so, rediscover the node and retry.
		if vclib.IsManagedObjectNotFoundError(err) {
			klog.V(4).Infof("error %q ManagedObjectNotFound for node %q. Rediscovering...", err, nodeVmDetail.NodeName)
			err = nodeManager.RediscoverNode(convertToK8sType(nodeVmDetail.NodeName))
			if err == nil {
				klog.V(4).Infof("Discovered node %s successfully", nodeVmDetail.NodeName)
				nodeInfo, err := nodeManager.GetNodeInfo(convertToK8sType(nodeVmDetail.NodeName))
				if err != nil {
					klog.V(4).Infof("error %q getting node info for node %+v", err, nodeVmDetail)
					return nil, err
				}

				accessibleDatastores, err = nodeInfo.vm.GetAllAccessibleDatastores(ctx)
				if err != nil {
					klog.V(4).Infof("error %q getting accessible datastores for node %+v", err, nodeVmDetail)
					return nil, err
				}
			} else {
				klog.V(4).Infof("error %q rediscovering node %+v", err, nodeVmDetail)
				return nil, err
			}
		} else {
			klog.V(4).Infof("error %q getting accessible datastores for node %+v", err, nodeVmDetail)
			return nil, err
		}
	}
	return accessibleDatastores, nil
}


// Get all datastores accessible for the virtual machine object.
func getSharedDatastoresInK8SCluster(ctx context.Context, nodeManager *NodeManager) ([]*vclib.DatastoreInfo, error) {
	nodeVmDetails, err := nodeManager.GetNodeDetails()
	if err != nil {
		klog.Errorf("Error while obtaining Kubernetes node nodeVmDetail details. error : %+v", err)
		return nil, err
	}

	if len(nodeVmDetails) == 0 {
		msg := fmt.Sprintf("Kubernetes node nodeVmDetail details is empty. nodeVmDetails : %+v", nodeVmDetails)
		klog.Error(msg)
		return nil, fmt.Errorf(msg)
	}
	var sharedDatastores []*vclib.DatastoreInfo
	for _, nodeVmDetail := range nodeVmDetails {
		klog.V(9).Infof("Getting accessible datastores for node %s", nodeVmDetail.NodeName)
		accessibleDatastores, err := getAccessibleDatastores(ctx, &nodeVmDetail, nodeManager)
		if err != nil {
			if err == vclib.ErrNoVMFound {
				klog.V(9).Infof("Got NoVMFound error for node %s", nodeVmDetail.NodeName)
				continue
			}
			return nil, err
		}

		if len(sharedDatastores) == 0 {
			sharedDatastores = accessibleDatastores
		} else {
			sharedDatastores = intersect(sharedDatastores, accessibleDatastores)
			if len(sharedDatastores) == 0 {
				return nil, fmt.Errorf("No shared datastores found in the Kubernetes cluster for nodeVmDetails: %+v", nodeVmDetails)
			}
		}
	}
	klog.V(9).Infof("sharedDatastores : %+v", sharedDatastores)
	return sharedDatastores, nil
}


func intersect(list1 []*vclib.DatastoreInfo, list2 []*vclib.DatastoreInfo) []*vclib.DatastoreInfo {
	klog.V(9).Infof("list1: %+v", list1)
	klog.V(9).Infof("list2: %+v", list2)
	var sharedDs []*vclib.DatastoreInfo
	for _, val1 := range list1 {
		// Check if val1 is found in list2
		for _, val2 := range list2 {
			// Intersection is performed based on the datastoreUrl as this uniquely identifies the datastore.
			if val1.Info.Url == val2.Info.Url {
				sharedDs = append(sharedDs, val1)
				break
			}
		}
	}
	return sharedDs
}


// getMostFreeDatastore gets the best fit compatible datastore by free space.
func getMostFreeDatastore(ctx context.Context, client *vim25.Client, dsInfoList []*vclib.DatastoreInfo) (*vclib.DatastoreInfo, error) {
	var curMax int64
	curMax = -1
	var index int
	for i, dsInfo := range dsInfoList {
		dsFreeSpace := dsInfo.Info.GetDatastoreInfo().FreeSpace
		if dsFreeSpace > curMax {
			curMax = dsFreeSpace
			index = i
		}
	}
	return dsInfoList[index], nil
}


func getPbmCompatibleDatastore(ctx context.Context, vcClient *vim25.Client, storagePolicyName string, nodeManager *NodeManager) (*vclib.DatastoreInfo, error) {
	pbmClient, err := vclib.NewPbmClient(ctx, vcClient)
	if err != nil {
		return nil, err
	}
	storagePolicyID, err := pbmClient.ProfileIDByName(ctx, storagePolicyName)
	if err != nil {
		klog.Errorf("Failed to get Profile ID by name: %s. err: %+v", storagePolicyName, err)
		return nil, err
	}
	sharedDs, err := getSharedDatastoresInK8SCluster(ctx, nodeManager)
	if err != nil {
		klog.Errorf("Failed to get shared datastores. err: %+v", err)
		return nil, err
	}
	if len(sharedDs) == 0 {
		msg := "No shared datastores found in the endpoint virtual center"
		klog.Errorf(msg)
		return nil, errors.New(msg)
	}
	compatibleDatastores, _, err := pbmClient.GetCompatibleDatastores(ctx, storagePolicyID, sharedDs)
	if err != nil {
		klog.Errorf("Failed to get compatible datastores from datastores : %+v with storagePolicy: %s. err: %+v",
			sharedDs, storagePolicyID, err)
		return nil, err
	}
	klog.V(9).Infof("compatibleDatastores : %+v", compatibleDatastores)
	datastore, err := getMostFreeDatastore(ctx, vcClient, compatibleDatastores)
	if err != nil {
		klog.Errorf("Failed to get most free datastore from compatible datastores: %+v. err: %+v", compatibleDatastores, err)
		return nil, err
	}
	klog.V(4).Infof("Most free datastore : %+s", datastore.Info.Name)
	return datastore, err
}


func getDatastoresForZone(ctx context.Context, nodeManager *NodeManager, selectedZones []string) ([]*vclib.DatastoreInfo, error) {

	var sharedDatastores []*vclib.DatastoreInfo

	for _, zone := range selectedZones {
		var sharedDatastoresPerZone []*vclib.DatastoreInfo
		hosts, err := nodeManager.GetHostsInZone(ctx, zone)
		if err != nil {
			return nil, err
		}
		klog.V(4).Infof("Hosts in zone %s : %s", zone, hosts)

		for _, host := range hosts {
			var hostSystemMo mo.HostSystem
			err = host.Properties(ctx, host.Reference(), []string{"datastore"}, &hostSystemMo)
			if err != nil {
				klog.Errorf("Failed to get datastore property for host %s. err : %+v", host, err)
				return nil, err
			}

			klog.V(4).Infof("Datastores mounted on host %s : %s", host, hostSystemMo.Datastore)
			var dsRefList []types.ManagedObjectReference
			for _, dsRef := range hostSystemMo.Datastore {
				dsRefList = append(dsRefList, dsRef)
			}

			var dsMoList []mo.Datastore
			pc := property.DefaultCollector(host.Client())
			properties := []string{DatastoreInfoProperty, DatastoreNameProperty}
			err = pc.Retrieve(ctx, dsRefList, properties, &dsMoList)
			if err != nil {
				klog.Errorf("Failed to get Datastore managed objects from datastore objects."+
					" dsObjList: %+v, properties: %+v, err: %+v", dsRefList, properties, err)
				return nil, err
			}
			klog.V(9).Infof("Datastore mo details: %+v", dsMoList)

			// find the Datacenter parent for this host
			mes, err := mo.Ancestors(ctx, host.Client(), pc.Reference(), host.Reference())
			if err != nil {
				return nil, err
			}
			var dcMoref *types.ManagedObjectReference
			for i := len(mes) - 1; i > 0; i-- {
				if mes[i].Self.Type == "Datacenter" {
					dcMoref = &mes[i].Self
					break
				}
			}
			if dcMoref == nil {
				return nil, fmt.Errorf("failed to find the Datacenter of host %s", host)
			}

			dc := object.NewDatacenter(host.Client(), *dcMoref)
			finder := find.NewFinder(host.Client(), false)
			finder.SetDatacenter(dc)
			var dsObjList []*vclib.DatastoreInfo
			for _, dsMo := range dsMoList {
				// use the finder so that InventoryPath is set correctly in dsObj
				dsObj, err := finder.Datastore(ctx, dsMo.Name)
				if err != nil {
					klog.Errorf("Failed to find datastore named %s in datacenter %s", dsMo.Name, dc)
					return nil, err
				}
				dsObjList = append(dsObjList,
					&vclib.DatastoreInfo{
						Datastore: &vclib.Datastore{Datastore: dsObj,
							Datacenter: &vclib.Datacenter{Datacenter: dc}},
						Info: dsMo.Info.GetDatastoreInfo()})
			}

			klog.V(9).Infof("DatastoreInfo details : %s", dsObjList)

			if len(sharedDatastoresPerZone) == 0 {
				sharedDatastoresPerZone = dsObjList
			} else {
				sharedDatastoresPerZone = intersect(sharedDatastoresPerZone, dsObjList)
				if len(sharedDatastoresPerZone) == 0 {
					klog.V(4).Infof("No shared datastores found among hosts %s", hosts)
					return nil, fmt.Errorf("No matching datastores found in the kubernetes cluster for zone %s", zone)
				}
			}
			klog.V(9).Infof("Shared datastore list after processing host %s : %s", host, sharedDatastoresPerZone)
		}
		klog.V(4).Infof("Shared datastore per zone %s is %s", zone, sharedDatastoresPerZone)
		if len(sharedDatastores) == 0 {
			sharedDatastores = sharedDatastoresPerZone
		} else {
			sharedDatastores = intersect(sharedDatastores, sharedDatastoresPerZone)
			if len(sharedDatastores) == 0 {
				return nil, fmt.Errorf("No matching datastores found in the kubernetes cluster across zones %s", selectedZones)
			}
		}
	}
	klog.V(1).Infof("Returning selected datastores : %s", sharedDatastores)
	return sharedDatastores, nil
}


func getPbmCompatibleZonedDatastore(ctx context.Context, vcClient *vim25.Client, storagePolicyName string, zonedDatastores []*vclib.DatastoreInfo) (*vclib.DatastoreInfo, error) {
	pbmClient, err := vclib.NewPbmClient(ctx, vcClient)
	if err != nil {
		return nil, err
	}
	storagePolicyID, err := pbmClient.ProfileIDByName(ctx, storagePolicyName)
	if err != nil {
		klog.Errorf("Failed to get Profile ID by name: %s. err: %+v", storagePolicyName, err)
		return nil, err
	}
	compatibleDatastores, _, err := pbmClient.GetCompatibleDatastores(ctx, storagePolicyID, zonedDatastores)
	if err != nil {
		klog.Errorf("Failed to get compatible datastores from datastores : %+v with storagePolicy: %s. err: %+v",
			zonedDatastores, storagePolicyID, err)
		return nil, err
	}
	klog.V(9).Infof("compatibleDatastores : %+v", compatibleDatastores)
	datastore, err := getMostFreeDatastore(ctx, vcClient, compatibleDatastores)
	if err != nil {
		klog.Errorf("Failed to get most free datastore from compatible datastores: %+v. err: %+v", compatibleDatastores, err)
		return nil, err
	}
	klog.V(4).Infof("Most free datastore : %+s", datastore.Info.Name)
	return datastore, err
}


func (vs *VSphere) setVMOptions(ctx context.Context, connection *vclib.VSphereConnection, ds *vclib.Datastore) (*vclib.VMOptions, error) {
	var vmOptions vclib.VMOptions
	dsHosts, err := ds.GetDatastoreHostMounts(ctx)
	if err != nil {
		klog.Errorf("Failed to get datastore host mounts for %v: %+v", ds, err)
		return nil, err
	}
	// pick a host at random to use for Volume creation
	dsHostMoref := dsHosts[rand.New(rand.NewSource(time.Now().UnixNano())).Intn(len(dsHosts))]
	dummyVMHost := object.NewHostSystem(connection.Client, dsHostMoref)
	resourcePool, err := dummyVMHost.ResourcePool(ctx)
	if err != nil {
		klog.Errorf("Failed to get resource pool from host %v", dummyVMHost)
		return nil, err
	}
	folder, err := ds.Datacenter.GetFolderByPath(ctx, vs.cfg.Workspace.Folder)
	if err != nil {
		return nil, err
	}
	vmOptions.VMFolder = folder
	vmOptions.VMResourcePool = resourcePool
	return &vmOptions, nil
}


// A background routine which will be responsible for deleting stale dummy VM's.
func (vs *VSphere) cleanUpDummyVMs(dummyVMPrefix string) {
	// Create context
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	for {
		time.Sleep(CleanUpDummyVMRoutineInterval * time.Minute)
		datacenters, err := vs.GetWorkspaceDatacenters(ctx)
		if err != nil {
			klog.V(4).Infof("Failed to get datacenters from VC. err: %+v", err)
			continue
		}
		// Clean up dummy VMs in each datacenter
		for _, dc := range datacenters {
			// Get the folder reference for global working directory where the dummy VM needs to be created.
			vmFolder, err := dc.GetFolderByPath(ctx, vs.cfg.Workspace.Folder)
			if err != nil {
				klog.V(4).Infof("Unable to get the kubernetes folder: %q reference. err: %+v", vs.cfg.Workspace.Folder, err)
				continue
			}
			// A write lock is acquired to make sure the cleanUp routine doesn't delete any VM's created by ongoing PVC requests.
			cleanUpDummyVMs := func() {
				cleanUpDummyVMLock.Lock()
				defer cleanUpDummyVMLock.Unlock()
				err = diskmanagers.CleanUpDummyVMs(ctx, vmFolder)
				if err != nil {
					klog.V(4).Infof("Unable to clean up dummy VM's in the kubernetes cluster: %q. err: %+v", vs.cfg.Workspace.Folder, err)
				}
			}
			cleanUpDummyVMs()
		}
	}
}


// Get canonical volume path for volume Path.
// Example1: The canonical path for volume path - [vsanDatastore] kubevols/volume.vmdk will be [vsanDatastore] 25d8b159-948c-4b73-e499-02001ad1b044/volume.vmdk
// Example2: The canonical path for volume path - [vsanDatastore] 25d8b159-948c-4b73-e499-02001ad1b044/volume.vmdk will be same as volume Path.
func getcanonicalVolumePath(ctx context.Context, dc *vclib.Datacenter, volumePath string) (string, error) {
	var folderID string
	var folderExists bool
	canonicalVolumePath := volumePath
	dsPathObj, err := vclib.GetDatastorePathObjFromVMDiskPath(volumePath)
	if err != nil {
		return "", err
	}
	dsPath := strings.Split(strings.TrimSpace(dsPathObj.Path), "/")
	if len(dsPath) <= 1 {
		return canonicalVolumePath, nil
	}
	datastore := dsPathObj.Datastore
	dsFolder := dsPath[0]
	folderNameIDMap, datastoreExists := datastoreFolderIDMap[datastore]
	if datastoreExists {
		folderID, folderExists = folderNameIDMap[dsFolder]
	}
	// Get the datastore folder ID if datastore or folder doesn't exist in datastoreFolderIDMap
	if !datastoreExists || !folderExists {
		if !vclib.IsValidUUID(dsFolder) {
			dummyDiskVolPath := "[" + datastore + "] " + dsFolder + "/" + DummyDiskName
			// Querying a non-existent dummy disk on the datastore folder.
			// It would fail and return an folder ID in the error message.
			_, err := dc.GetVirtualDiskPage83Data(ctx, dummyDiskVolPath)
			canonicalVolumePath, err = getPathFromFileNotFound(err)
			if err != nil {
				return "", fmt.Errorf("failed to get path from dummy request: %v", err)
			}
		}
		diskPath := vclib.GetPathFromVMDiskPath(canonicalVolumePath)
		if diskPath == "" {
			return "", fmt.Errorf("Failed to parse canonicalVolumePath: %s in getcanonicalVolumePath method", canonicalVolumePath)
		}
		folderID = strings.Split(strings.TrimSpace(diskPath), "/")[0]
		setdatastoreFolderIDMap(datastoreFolderIDMap, datastore, dsFolder, folderID)
	}
	canonicalVolumePath = strings.Replace(volumePath, dsFolder, folderID, 1)
	return canonicalVolumePath, nil
}


// getPathFromFileNotFound returns the path from a fileNotFound error
func getPathFromFileNotFound(err error) (string, error) {
	if soap.IsSoapFault(err) {
		fault := soap.ToSoapFault(err)
		f, ok := fault.VimFault().(types.FileNotFound)
		if !ok {
			return "", fmt.Errorf("%v is not a FileNotFound error", err)
		}
		return f.File, nil
	}
	return "", fmt.Errorf("%v is not a soap fault", err)
}


func setdatastoreFolderIDMap(

func setdatastoreFolderIDMap(
	datastoreFolderIDMap map[string]map[string]string,
	datastore string,
	folderName string,
	folderID string) {
	folderNameIDMap := datastoreFolderIDMap[datastore]
	if folderNameIDMap == nil {
		folderNameIDMap = make(map[string]string)
		datastoreFolderIDMap[datastore] = folderNameIDMap
	}
	folderNameIDMap[folderName] = folderID
}


func convertVolPathToDevicePath(ctx context.Context, dc *vclib.Datacenter, volPath string) (string, error) {
	volPath = vclib.RemoveStorageClusterORFolderNameFromVDiskPath(volPath)
	// Get the canonical volume path for volPath.
	canonicalVolumePath, err := getcanonicalVolumePath(ctx, dc, volPath)
	if err != nil {
		klog.Errorf("Failed to get canonical vsphere volume path for volume: %s. err: %+v", volPath, err)
		return "", err
	}
	// Check if the volume path contains .vmdk extension. If not, add the extension and update the nodeVolumes Map
	if len(canonicalVolumePath) > 0 && filepath.Ext(canonicalVolumePath) != ".vmdk" {
		canonicalVolumePath += ".vmdk"
	}
	return canonicalVolumePath, nil
}


// convertVolPathsToDevicePaths removes cluster or folder path from volPaths and convert to canonicalPath
func (vs *VSphere) convertVolPathsToDevicePaths(ctx context.Context, nodeVolumes map[k8stypes.NodeName][]string) (map[k8stypes.NodeName][]string, error) {
	vmVolumes := make(map[k8stypes.NodeName][]string)
	for nodeName, volPaths := range nodeVolumes {
		nodeInfo, err := vs.nodeManager.GetNodeInfo(nodeName)
		if err != nil {
			return nil, err
		}

		_, err = vs.getVSphereInstanceForServer(nodeInfo.vcServer, ctx)
		if err != nil {
			return nil, err
		}

		for i, volPath := range volPaths {
			deviceVolPath, err := convertVolPathToDevicePath(ctx, nodeInfo.dataCenter, volPath)
			if err != nil {
				klog.Errorf("Failed to convert vsphere volume path %s to device path for volume %s. err: %+v", volPath, deviceVolPath, err)
				return nil, err
			}
			volPaths[i] = deviceVolPath
		}
		vmVolumes[nodeName] = volPaths
	}
	return vmVolumes, nil
}


// checkDiskAttached verifies volumes are attached to the VMs which are in same vCenter and Datacenter
// Returns nodes if exist any for which VM is not found in that vCenter and Datacenter
func (vs *VSphere) checkDiskAttached(ctx context.Context, nodes []k8stypes.NodeName, nodeVolumes map[k8stypes.NodeName][]string, attached map[string]map[string]bool, retry bool) ([]k8stypes.NodeName, error) {
	var nodesToRetry []k8stypes.NodeName
	var vmList []*vclib.VirtualMachine
	var nodeInfo NodeInfo
	var err error

	for _, nodeName := range nodes {
		nodeInfo, err = vs.nodeManager.GetNodeInfo(nodeName)
		if err != nil {
			return nodesToRetry, err
		}
		vmList = append(vmList, nodeInfo.vm)
	}

	// Making sure session is valid
	_, err = vs.getVSphereInstanceForServer(nodeInfo.vcServer, ctx)
	if err != nil {
		return nodesToRetry, err
	}

	// If any of the nodes are not present property collector query will fail for entire operation
	vmMoList, err := nodeInfo.dataCenter.GetVMMoList(ctx, vmList, []string{"config.hardware.device", "name", "config.uuid"})
	if err != nil {
		if vclib.IsManagedObjectNotFoundError(err) && !retry {
			klog.V(4).Infof("checkDiskAttached: ManagedObjectNotFound for property collector query for nodes: %+v vms: %+v", nodes, vmList)
			// Property Collector Query failed
			// VerifyVolumePaths per VM
			for _, nodeName := range nodes {
				nodeInfo, err := vs.nodeManager.GetNodeInfo(nodeName)
				if err != nil {
					return nodesToRetry, err
				}
				devices, err := nodeInfo.vm.VirtualMachine.Device(ctx)
				if err != nil {
					if vclib.IsManagedObjectNotFoundError(err) {
						klog.V(4).Infof("checkDiskAttached: ManagedObjectNotFound for Kubernetes node: %s with vSphere Virtual Machine reference: %v", nodeName, nodeInfo.vm)
						nodesToRetry = append(nodesToRetry, nodeName)
						continue
					}
					return nodesToRetry, err
				}
				klog.V(4).Infof("Verifying Volume Paths by devices for node %s and VM %s", nodeName, nodeInfo.vm)
				vs.vsphereVolumeMap.Add(nodeName, devices)
				vclib.VerifyVolumePathsForVMDevices(devices, nodeVolumes[nodeName], convertToString(nodeName), attached)
			}
		}
		return nodesToRetry, err
	}

	vmMoMap := make(map[string]mo.VirtualMachine)
	for _, vmMo := range vmMoList {
		if vmMo.Config == nil {
			klog.Errorf("Config is not available for VM: %q", vmMo.Name)
			continue
		}
		klog.V(9).Infof("vmMoMap vmname: %q vmuuid: %s", vmMo.Name, strings.ToLower(vmMo.Config.Uuid))
		vmMoMap[strings.ToLower(vmMo.Config.Uuid)] = vmMo
	}

	klog.V(9).Infof("vmMoMap: +%v", vmMoMap)

	for _, nodeName := range nodes {
		node, err := vs.nodeManager.GetNode(nodeName)
		if err != nil {
			return nodesToRetry, err
		}
		nodeUUID, err := GetNodeUUID(&node)
		if err != nil {
			klog.Errorf("Node Discovery failed to get node uuid for node %s with error: %v", node.Name, err)
			return nodesToRetry, err
		}
		nodeUUID = strings.ToLower(nodeUUID)
		klog.V(9).Infof("Verifying volume for node %s with nodeuuid %q: %v", nodeName, nodeUUID, vmMoMap)
		vmMo := vmMoMap[nodeUUID]
		vmDevices := object.VirtualDeviceList(vmMo.Config.Hardware.Device)
		vs.vsphereVolumeMap.Add(nodeName, vmDevices)
		vclib.VerifyVolumePathsForVMDevices(vmDevices, nodeVolumes[nodeName], convertToString(nodeName), attached)
	}
	return nodesToRetry, nil
}


// BuildMissingVolumeNodeMap builds a map of volumes and nodes which are not known to attach detach controller.
// There could be nodes in cluster which do not have any pods with vsphere volumes running on them
// such nodes won't be part of disk verification check because attach-detach controller does not keep track
// such nodes. But such nodes may still have dangling volumes on them and hence we need to scan all the
// remaining nodes which weren't scanned by code previously.
func (vs *VSphere) BuildMissingVolumeNodeMap(ctx context.Context) {
	nodeNames := vs.nodeManager.GetNodeNames()
	// Segregate nodes according to VC-DC
	dcNodes := make(map[string][]k8stypes.NodeName)

	for _, nodeName := range nodeNames {
		// if given node is not in node volume map
		if !vs.vsphereVolumeMap.CheckForNode(nodeName) {
			nodeInfo, err := vs.nodeManager.GetNodeInfo(nodeName)
			if err != nil {
				klog.V(4).Infof("Failed to get node info: %+v. err: %+v", nodeInfo.vm, err)
				continue
			}
			vcDC := nodeInfo.vcServer + nodeInfo.dataCenter.String()
			dcNodes[vcDC] = append(dcNodes[vcDC], nodeName)
		}
	}

	var wg sync.WaitGroup

	for _, nodeNames := range dcNodes {
		// Start go routines per VC-DC to check disks are attached
		wg.Add(1)
		go func(nodes []k8stypes.NodeName) {
			err := vs.checkNodeDisks(ctx, nodes)
			if err != nil {
				klog.Errorf("Failed to check disk attached for nodes: %+v. err: %+v", nodes, err)
			}
			wg.Done()
		}(nodeNames)
	}
	wg.Wait()
}


func (vs *VSphere) checkNodeDisks(ctx context.Context, nodeNames []k8stypes.NodeName) error {
	var vmList []*vclib.VirtualMachine
	var nodeInfo NodeInfo
	var err error

	for _, nodeName := range nodeNames {
		nodeInfo, err = vs.nodeManager.GetNodeInfo(nodeName)
		if err != nil {
			return err
		}
		vmList = append(vmList, nodeInfo.vm)
	}

	// Making sure session is valid
	_, err = vs.getVSphereInstanceForServer(nodeInfo.vcServer, ctx)
	if err != nil {
		return err
	}

	// If any of the nodes are not present property collector query will fail for entire operation
	vmMoList, err := nodeInfo.dataCenter.GetVMMoList(ctx, vmList, []string{"config.hardware.device", "name", "config.uuid"})
	if err != nil {
		if vclib.IsManagedObjectNotFoundError(err) {
			klog.V(4).Infof("checkNodeDisks: ManagedObjectNotFound for property collector query for nodes: %+v vms: %+v", nodeNames, vmList)
			// Property Collector Query failed
			// VerifyVolumePaths per VM
			for _, nodeName := range nodeNames {
				nodeInfo, err := vs.nodeManager.GetNodeInfo(nodeName)
				if err != nil {
					return err
				}
				devices, err := nodeInfo.vm.VirtualMachine.Device(ctx)
				if err != nil {
					if vclib.IsManagedObjectNotFoundError(err) {
						klog.V(4).Infof("checkNodeDisks: ManagedObjectNotFound for Kubernetes node: %s with vSphere Virtual Machine reference: %v", nodeName, nodeInfo.vm)
						continue
					}
					return err
				}
				klog.V(4).Infof("Verifying Volume Paths by devices for node %s and VM %s", nodeName, nodeInfo.vm)
				vs.vsphereVolumeMap.Add(nodeName, devices)
			}
			return nil
		}
		return err
	}

	vmMoMap := make(map[string]mo.VirtualMachine)
	for _, vmMo := range vmMoList {
		if vmMo.Config == nil {
			klog.Errorf("Config is not available for VM: %q", vmMo.Name)
			continue
		}
		klog.V(9).Infof("vmMoMap vmname: %q vmuuid: %s", vmMo.Name, strings.ToLower(vmMo.Config.Uuid))
		vmMoMap[strings.ToLower(vmMo.Config.Uuid)] = vmMo
	}

	klog.V(9).Infof("vmMoMap: +%v", vmMoMap)

	for _, nodeName := range nodeNames {
		node, err := vs.nodeManager.GetNode(nodeName)
		if err != nil {
			return err
		}
		nodeUUID, err := GetNodeUUID(&node)
		if err != nil {
			klog.Errorf("Node Discovery failed to get node uuid for node %s with error: %v", node.Name, err)
			return err
		}
		nodeUUID = strings.ToLower(nodeUUID)
		klog.V(9).Infof("Verifying volume for node %s with nodeuuid %q: %v", nodeName, nodeUUID, vmMoMap)
		vmMo := vmMoMap[nodeUUID]
		vmDevices := object.VirtualDeviceList(vmMo.Config.Hardware.Device)
		vs.vsphereVolumeMap.Add(nodeName, vmDevices)
	}
	return nil
}


func (vs *VSphere) GetNodeNameFromProviderID(providerID string) (string, error) {
	var nodeName string
	nodes, err := vs.nodeManager.GetNodeDetails()
	if err != nil {
		klog.Errorf("Error while obtaining Kubernetes node nodeVmDetail details. error : %+v", err)
		return "", err
	}
	for _, node := range nodes {
		// ProviderID is UUID for nodes v1.9.3+
		if node.VMUUID == GetUUIDFromProviderID(providerID) || node.NodeName == providerID {
			nodeName = node.NodeName
			break
		}
	}
	if nodeName == "" {
		msg := fmt.Sprintf("Error while obtaining Kubernetes nodename for providerID %s.", providerID)
		return "", errors.New(msg)
	}
	return nodeName, nil
}


func GetUUIDFromProviderID(providerID string) string {
	return strings.TrimPrefix(providerID, ProviderPrefix)
}


func IsUUIDSupportedNode(node *v1.Node) (bool, error) {
	newVersion, err := version.ParseSemantic("v1.9.4")
	if err != nil {
		klog.Errorf("Failed to determine whether node %+v is old with error %v", node, err)
		return false, err
	}
	nodeVersion, err := version.ParseSemantic(node.Status.NodeInfo.KubeletVersion)
	if err != nil {
		klog.Errorf("Failed to determine whether node %+v is old with error %v", node, err)
		return false, err
	}
	if nodeVersion.LessThan(newVersion) {
		return true, nil
	}
	return false, nil
}


func isGuestHardwareVersionDeprecated(vmHardwareversion string) (bool, error) {
	vmHardwareDeprecated := false
	// vmconfig.Version returns vm hardware version as vmx-11, vmx-13, vmx-14, vmx-15 etc.
	version := strings.Trim(vmHardwareversion, "vmx-")
	value, err := strconv.ParseInt(version, 0, 64)
	if err != nil {
		return false, fmt.Errorf("failed to parse vm hardware version: %v Err: %v", version, err)
	} else {
		if value < 15 {
			vmHardwareDeprecated = true
		}
	}
	return vmHardwareDeprecated, nil
}


func GetNodeUUID(node *v1.Node) (string, error) {
	oldNode, err := IsUUIDSupportedNode(node)
	if err != nil {
		klog.Errorf("Failed to get node UUID for node %+v with error %v", node, err)
		return "", err
	}
	if oldNode {
		return node.Status.NodeInfo.SystemUUID, nil
	}
	return GetUUIDFromProviderID(node.Spec.ProviderID), nil
}


func GetVMUUID() (string, error)

func GetVMUUID() (string, error) {
	uuidFromFile, err := getRawUUID()
	if err != nil {
		return "", fmt.Errorf("error retrieving vm uuid: %s", err)
	}
	//strip leading and trailing white space and new line char
	uuid := strings.TrimSpace(uuidFromFile)
	// check the uuid starts with "VMware-"
	if !strings.HasPrefix(uuid, UUIDPrefix) {
		return "", fmt.Errorf("failed to match Prefix, UUID read from the file is %v", uuidFromFile)
	}
	// Strip the prefix and white spaces and -
	uuid = strings.Replace(uuid[len(UUIDPrefix):(len(uuid))], " ", "", -1)
	uuid = strings.Replace(uuid, "-", "", -1)
	if len(uuid) != 32 {
		return "", fmt.Errorf("length check failed, UUID read from the file is %v", uuidFromFile)
	}
	// need to add dashes, e.g. "564d395e-d807-e18a-cb25-b79f65eb2b9f"
	uuid = fmt.Sprintf("%s-%s-%s-%s-%s", uuid[0:8], uuid[8:12], uuid[12:16], uuid[16:20], uuid[20:32])
	return uuid, nil
}


// GetWorkspaceDatacenters returns the Datacenter objects that VCP has access to.
// User can configure the list of datacenters in vsphere.conf. Otherwise all the
// Datacenters in the configured list of VCs are returned.
func (vs *VSphere) GetWorkspaceDatacenters(ctx context.Context) ([]*vclib.Datacenter, error) {
	var datacenterObjs []*vclib.Datacenter
	for vc, vsi := range vs.vsphereInstanceMap {
		// ensure connection to VC
		err := vs.nodeManager.vcConnect(ctx, vsi)
		if err != nil {
			return nil, err
		}
		if vsi.cfg.Datacenters == "" {
			vcDatacenterObjs, err := vclib.GetAllDatacenter(ctx, vsi.conn)
			if err != nil {
				klog.Errorf("Error fetching list of datacenters from VC %s: %+v", vc, err)
				return nil, err
			}
			datacenterObjs = append(datacenterObjs, vcDatacenterObjs...)
		} else {
			datacenters := strings.Split(vsi.cfg.Datacenters, ",")
			for _, dc := range datacenters {
				dc = strings.TrimSpace(dc)
				if dc == "" {
					continue
				}
				datacenterObj, err := vclib.GetDatacenter(ctx, vsi.conn, dc)
				if err != nil {
					klog.Errorf("Error fetching datacenter %s from VC %s: %+v", dc, vc, err)
					return nil, err
				}
				datacenterObjs = append(datacenterObjs, datacenterObj)
			}
		}
	}
	return datacenterObjs, nil
}


// FindDatastoreByName looks for the given datastore by name across all available datacenters.
// If more than one Datacenter has a Datastore with the given name, then returns reference to all of them.
func (vs *VSphere) FindDatastoreByName(ctx context.Context, datastoreName string) ([]*vclib.DatastoreInfo, error) {
	datacenters, err := vs.GetWorkspaceDatacenters(ctx)
	if err != nil {
		return nil, err
	}
	var datastoreInfos []*vclib.DatastoreInfo
	for _, dc := range datacenters {
		datastoreInfo, err := dc.GetDatastoreInfoByName(ctx, datastoreName)
		if err != nil {
			klog.V(9).Infof("Did not find datastore %s in datacenter %s, still looking.", datastoreName, dc.Name())
			continue
		}
		datastoreInfos = append(datastoreInfos, datastoreInfo)
	}
	if len(datastoreInfos) == 0 {
		return nil, fmt.Errorf("Datastore '%s' not found", datastoreName)
	}
	klog.V(4).Infof("Found datastore infos %v for datastore %s", datastoreInfos, datastoreName)
	return datastoreInfos, nil
}


func getRawUUID() (string, error)

func getRawUUID() (string, error) {
	id, err := ioutil.ReadFile(UUIDPath)
	if err != nil {
		return "", err
	}
	return string(id), nil
}


func getRawUUID() (string, error) {
	return "", fmt.Errorf("Retrieving VM UUID on this build is not implemented.")
}


func getRawUUID() (string, error) {
	result, err := exec.Command("wmic", "bios", "get", "serialnumber").Output()
	if err != nil {
		return "", err
	}
	lines := strings.FieldsFunc(string(result), func(r rune) bool {
		switch r {
		case '\n', '\r':
			return true
		default:
			return false
		}
	})
	if len(lines) != 2 {
		return "", fmt.Errorf("received unexpected value retrieving vm uuid: %q", string(result))
	}
	return lines[1], nil
}


func NewVsphereVolumeMap() *VsphereVolumeMap {
	return &VsphereVolumeMap{
		volumeNodeMap: map[volumePath]nodeVolumeStatus{},
		nodeMap:       map[k8stypes.NodeName]bool{},
	}
}


// StartDiskVerification marks all known volumes as unverified so as
// disks which aren't verified can be removed at the end of verification process
func (vsphereVolume *VsphereVolumeMap) StartDiskVerification() {
	vsphereVolume.lock.Lock()
	defer vsphereVolume.lock.Unlock()
	for k, v := range vsphereVolume.volumeNodeMap {
		v.verified = false
		vsphereVolume.volumeNodeMap[k] = v
	}
	// reset nodeMap to empty so that any node we could not verify via usual verification process
	// can still be verified.
	vsphereVolume.nodeMap = map[k8stypes.NodeName]bool{}
}


// CheckForVolume verifies if disk is attached to some node in the cluster.
// This check is not definitive and should be followed up by separate verification.
func (vsphereVolume *VsphereVolumeMap) CheckForVolume(path string) (k8stypes.NodeName, bool) {
	vsphereVolume.lock.RLock()
	defer vsphereVolume.lock.RUnlock()
	vPath := volumePath(path)
	ns, ok := vsphereVolume.volumeNodeMap[vPath]
	if ok {
		return ns.nodeName, true
	}
	return "", false
}


// CheckForNode returns true if given node has already been processed by volume
// verification mechanism. This is used to skip verifying attached disks on nodes
// which were previously verified.
func (vsphereVolume *VsphereVolumeMap) CheckForNode(nodeName k8stypes.NodeName) bool {
	vsphereVolume.lock.RLock()
	defer vsphereVolume.lock.RUnlock()
	_, ok := vsphereVolume.nodeMap[nodeName]
	return ok
}


// Add all devices found on a node to the device map
func (vsphereVolume *VsphereVolumeMap) Add(node k8stypes.NodeName, vmDevices object.VirtualDeviceList) {
	vsphereVolume.lock.Lock()
	defer vsphereVolume.lock.Unlock()
	for _, device := range vmDevices {
		if vmDevices.TypeName(device) == "VirtualDisk" {
			virtualDevice := device.GetVirtualDevice()
			if backing, ok := virtualDevice.Backing.(*types.VirtualDiskFlatVer2BackingInfo); ok {
				filename := volumePath(backing.FileName)
				vsphereVolume.volumeNodeMap[filename] = nodeVolumeStatus{node, true}
				vsphereVolume.nodeMap[node] = true
			}
		}
	}
}


// RemoveUnverified will remove any device which we could not verify to be attached to a node.
func (vsphereVolume *VsphereVolumeMap) RemoveUnverified() {
	vsphereVolume.lock.Lock()
	defer vsphereVolume.lock.Unlock()
	for k, v := range vsphereVolume.volumeNodeMap {
		if !v.verified {
			delete(vsphereVolume.volumeNodeMap, k)
			delete(vsphereVolume.nodeMap, v.nodeName)
		}
	}
}


func Convert_v1_ObjectReference_To_custom_metrics_ObjectReference(in *v1.ObjectReference, out *ObjectReference, s conversion.Scope) error {
	out.APIVersion = in.APIVersion

	out.Kind = in.Kind
	out.Namespace = in.Namespace
	out.Name = in.Name
	out.UID = in.UID
	out.ResourceVersion = in.ResourceVersion
	out.FieldPath = in.FieldPath
	return nil
}


func Convert_custom_metrics_ObjectReference_To_v1_ObjectReference(in *ObjectReference, out *v1.ObjectReference, s conversion.Scope) error {
	out.APIVersion = in.APIVersion

	out.Kind = in.Kind
	out.Namespace = in.Namespace
	out.Name = in.Name
	out.UID = in.UID
	out.ResourceVersion = in.ResourceVersion
	out.FieldPath = in.FieldPath
	return nil
}


func Install(scheme *runtime.Scheme)

// Install registers the API group and adds types to a scheme
func Install(scheme *runtime.Scheme) {
	utilruntime.Must(custom_metrics.AddToScheme(scheme))
	utilruntime.Must(v1beta2.AddToScheme(scheme))
	utilruntime.Must(v1beta1.AddToScheme(scheme))
	utilruntime.Must(scheme.SetVersionPriority(v1beta1.SchemeGroupVersion, v1beta2.SchemeGroupVersion))
}


func Kind(kind string) schema.GroupKind

// Kind takes an unqualified kind and returns back a Group qualified GroupKind
func Kind(kind string) schema.GroupKind {
	return SchemeGroupVersion.WithKind(kind).GroupKind()
}


// Resource takes an unqualified resource and returns back a Group qualified GroupResource
func Resource(resource string) schema.GroupResource {
	return SchemeGroupVersion.WithResource(resource).GroupResource()
}


func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&MetricValue{},
		&MetricValueList{},
		&MetricListOptions{},
	)
	return nil
}


func Convert_v1beta1_MetricValue_To_custom_metrics_MetricValue(in *MetricValue, out *custom_metrics.MetricValue, s conversion.Scope) error {
	if err := autoConvert_v1beta1_MetricValue_To_custom_metrics_MetricValue(in, out, s); err != nil {
		return err
	}
	out.Metric.Name = in.MetricName
	out.Metric.Selector = in.Selector
	return nil
}


func Convert_custom_metrics_MetricValue_To_v1beta1_MetricValue(in *custom_metrics.MetricValue, out *MetricValue, s conversion.Scope) error {
	if err := autoConvert_custom_metrics_MetricValue_To_v1beta1_MetricValue(in, out, s); err != nil {
		return err
	}
	out.MetricName = in.Metric.Name
	out.Selector = in.Metric.Selector
	return nil
}


// Resource takes an unqualified resource and returns a Group qualified GroupResource
func Resource(resource string) schema.GroupResource {
	return SchemeGroupVersion.WithResource(resource).GroupResource()
}


func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&MetricValue{},
		&MetricValueList{},
		&MetricListOptions{},
	)
	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
	return nil
}


func Install(scheme *runtime.Scheme)

// Install registers the API group and adds types to a scheme
func Install(scheme *runtime.Scheme) {
	utilruntime.Must(external_metrics.AddToScheme(scheme))
	utilruntime.Must(v1beta1.AddToScheme(scheme))
	utilruntime.Must(scheme.SetVersionPriority(v1beta1.SchemeGroupVersion))
}


func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&ExternalMetricValue{},
		&ExternalMetricValueList{},
	)
	return nil
}


func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&ExternalMetricValue{},
		&ExternalMetricValueList{},
	)
	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
	return nil
}


// Install registers the API group and adds types to a scheme
func Install(scheme *runtime.Scheme) {
	utilruntime.Must(metrics.AddToScheme(scheme))
	utilruntime.Must(v1beta1.AddToScheme(scheme))
	utilruntime.Must(scheme.SetVersionPriority(v1beta1.SchemeGroupVersion))
}


func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&NodeMetrics{},
		&NodeMetricsList{},
		&PodMetrics{},
		&PodMetricsList{},
	)
	return nil
}


func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&NodeMetrics{},
		&NodeMetricsList{},
		&PodMetrics{},
		&PodMetricsList{},
	)
	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
	return nil
}


// MetricsV1alpha1 retrieves the MetricsV1alpha1Client
func (c *Clientset) MetricsV1alpha1() metricsv1alpha1.MetricsV1alpha1Interface {
	return c.metricsV1alpha1
}


// MetricsV1beta1 retrieves the MetricsV1beta1Client
func (c *Clientset) MetricsV1beta1() metricsv1beta1.MetricsV1beta1Interface {
	return c.metricsV1beta1
}


// Discovery retrieves the DiscoveryClient
func (c *Clientset) Discovery() discovery.DiscoveryInterface {
	if c == nil {
		return nil
	}
	return c.DiscoveryClient
}


// NewForConfig creates a new Clientset for the given config.
// If config's RateLimiter is not set and QPS and Burst are acceptable,
// NewForConfig will generate a rate-limiter in configShallowCopy.
// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient),
// where httpClient was generated with rest.HTTPClientFor(c).
func NewForConfig(c *rest.Config) (*Clientset, error) {
	configShallowCopy := *c

	if configShallowCopy.UserAgent == "" {
		configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent()
	}

	// share the transport between all clients
	httpClient, err := rest.HTTPClientFor(&configShallowCopy)
	if err != nil {
		return nil, err
	}

	return NewForConfigAndClient(&configShallowCopy, httpClient)
}


// NewForConfigAndClient creates a new Clientset for the given config and http client.
// Note the http client provided takes precedence over the configured transport values.
// If config's RateLimiter is not set and QPS and Burst are acceptable,
// NewForConfigAndClient will generate a rate-limiter in configShallowCopy.
func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, error) {
	configShallowCopy := *c
	if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 {
		if configShallowCopy.Burst <= 0 {
			return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0")
		}
		configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst)
	}

	var cs Clientset
	var err error
	cs.metricsV1alpha1, err = metricsv1alpha1.NewForConfigAndClient(&configShallowCopy, httpClient)
	if err != nil {
		return nil, err
	}
	cs.metricsV1beta1, err = metricsv1beta1.NewForConfigAndClient(&configShallowCopy, httpClient)
	if err != nil {
		return nil, err
	}

	cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient)
	if err != nil {
		return nil, err
	}
	return &cs, nil
}


// NewForConfigOrDie creates a new Clientset for the given config and
// panics if there is an error in the config.
func NewForConfigOrDie(c *rest.Config) *Clientset {
	cs, err := NewForConfig(c)
	if err != nil {
		panic(err)
	}
	return cs
}


func New(c rest.Interface) *Clientset

// New creates a new Clientset for the given RESTClient.
func New(c rest.Interface) *Clientset {
	var cs Clientset
	cs.metricsV1alpha1 = metricsv1alpha1.New(c)
	cs.metricsV1beta1 = metricsv1beta1.New(c)

	cs.DiscoveryClient = discovery.NewDiscoveryClient(c)
	return &cs
}


func init()

func init() {
	v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"})
	utilruntime.Must(AddToScheme(Scheme))
}


func (c *MetricsV1alpha1Client) NodeMetricses() NodeMetricsInterface {
	return newNodeMetricses(c)
}


func (c *MetricsV1alpha1Client) PodMetricses(namespace string) PodMetricsInterface {
	return newPodMetricses(c, namespace)
}


// NewForConfig creates a new MetricsV1alpha1Client for the given config.
// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient),
// where httpClient was generated with rest.HTTPClientFor(c).
func NewForConfig(c *rest.Config) (*MetricsV1alpha1Client, error) {
	config := *c
	if err := setConfigDefaults(&config); err != nil {
		return nil, err
	}
	httpClient, err := rest.HTTPClientFor(&config)
	if err != nil {
		return nil, err
	}
	return NewForConfigAndClient(&config, httpClient)
}


// NewForConfigAndClient creates a new MetricsV1alpha1Client for the given config and http client.
// Note the http client provided takes precedence over the configured transport values.
func NewForConfigAndClient(c *rest.Config, h *http.Client) (*MetricsV1alpha1Client, error) {
	config := *c
	if err := setConfigDefaults(&config); err != nil {
		return nil, err
	}
	client, err := rest.RESTClientForConfigAndClient(&config, h)
	if err != nil {
		return nil, err
	}
	return &MetricsV1alpha1Client{client}, nil
}


// NewForConfigOrDie creates a new MetricsV1alpha1Client for the given config and
// panics if there is an error in the config.
func NewForConfigOrDie(c *rest.Config) *MetricsV1alpha1Client {
	client, err := NewForConfig(c)
	if err != nil {
		panic(err)
	}
	return client
}


// New creates a new MetricsV1alpha1Client for the given RESTClient.
func New(c rest.Interface) *MetricsV1alpha1Client {
	return &MetricsV1alpha1Client{c}
}


func setConfigDefaults(config *rest.Config) error {
	gv := v1alpha1.SchemeGroupVersion
	config.GroupVersion = &gv
	config.APIPath = "/apis"
	config.NegotiatedSerializer = scheme.Codecs.WithoutConversion()

	if config.UserAgent == "" {
		config.UserAgent = rest.DefaultKubernetesUserAgent()
	}

	return nil
}


// RESTClient returns a RESTClient that is used to communicate
// with API server by this client implementation.
func (c *MetricsV1alpha1Client) RESTClient() rest.Interface {
	if c == nil {
		return nil
	}
	return c.restClient
}


// newNodeMetricses returns a NodeMetricses
func newNodeMetricses(c *MetricsV1alpha1Client) *nodeMetricses {
	return &nodeMetricses{
		client: c.RESTClient(),
	}
}


// Get takes name of the nodeMetrics, and returns the corresponding nodeMetrics object, and an error if there is any.
func (c *nodeMetricses) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.NodeMetrics, err error) {
	result = &v1alpha1.NodeMetrics{}
	err = c.client.Get().
		Resource("nodes").
		Name(name).
		VersionedParams(&options, scheme.ParameterCodec).
		Do(ctx).
		Into(result)
	return
}


// List takes label and field selectors, and returns the list of NodeMetricses that match those selectors.
func (c *nodeMetricses) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.NodeMetricsList, err error) {
	var timeout time.Duration
	if opts.TimeoutSeconds != nil {
		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
	}
	result = &v1alpha1.NodeMetricsList{}
	err = c.client.Get().
		Resource("nodes").
		VersionedParams(&opts, scheme.ParameterCodec).
		Timeout(timeout).
		Do(ctx).
		Into(result)
	return
}


// Watch returns a watch.Interface that watches the requested nodeMetricses.
func (c *nodeMetricses) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
	var timeout time.Duration
	if opts.TimeoutSeconds != nil {
		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
	}
	opts.Watch = true
	return c.client.Get().
		Resource("nodes").
		VersionedParams(&opts, scheme.ParameterCodec).
		Timeout(timeout).
		Watch(ctx)
}


// newPodMetricses returns a PodMetricses
func newPodMetricses(c *MetricsV1alpha1Client, namespace string) *podMetricses {
	return &podMetricses{
		client: c.RESTClient(),
		ns:     namespace,
	}
}


// Get takes name of the podMetrics, and returns the corresponding podMetrics object, and an error if there is any.
func (c *podMetricses) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.PodMetrics, err error) {
	result = &v1alpha1.PodMetrics{}
	err = c.client.Get().
		Namespace(c.ns).
		Resource("pods").
		Name(name).
		VersionedParams(&options, scheme.ParameterCodec).
		Do(ctx).
		Into(result)
	return
}


// List takes label and field selectors, and returns the list of PodMetricses that match those selectors.
func (c *podMetricses) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.PodMetricsList, err error) {
	var timeout time.Duration
	if opts.TimeoutSeconds != nil {
		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
	}
	result = &v1alpha1.PodMetricsList{}
	err = c.client.Get().
		Namespace(c.ns).
		Resource("pods").
		VersionedParams(&opts, scheme.ParameterCodec).
		Timeout(timeout).
		Do(ctx).
		Into(result)
	return
}


// Watch returns a watch.Interface that watches the requested podMetricses.
func (c *podMetricses) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
	var timeout time.Duration
	if opts.TimeoutSeconds != nil {
		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
	}
	opts.Watch = true
	return c.client.Get().
		Namespace(c.ns).
		Resource("pods").
		VersionedParams(&opts, scheme.ParameterCodec).
		Timeout(timeout).
		Watch(ctx)
}


func (c *MetricsV1beta1Client) NodeMetricses() NodeMetricsInterface {
	return newNodeMetricses(c)
}


func (c *MetricsV1beta1Client) PodMetricses(namespace string) PodMetricsInterface {
	return newPodMetricses(c, namespace)
}


// NewForConfig creates a new MetricsV1beta1Client for the given config.
// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient),
// where httpClient was generated with rest.HTTPClientFor(c).
func NewForConfig(c *rest.Config) (*MetricsV1beta1Client, error) {
	config := *c
	if err := setConfigDefaults(&config); err != nil {
		return nil, err
	}
	httpClient, err := rest.HTTPClientFor(&config)
	if err != nil {
		return nil, err
	}
	return NewForConfigAndClient(&config, httpClient)
}


// NewForConfigAndClient creates a new MetricsV1beta1Client for the given config and http client.
// Note the http client provided takes precedence over the configured transport values.
func NewForConfigAndClient(c *rest.Config, h *http.Client) (*MetricsV1beta1Client, error) {
	config := *c
	if err := setConfigDefaults(&config); err != nil {
		return nil, err
	}
	client, err := rest.RESTClientForConfigAndClient(&config, h)
	if err != nil {
		return nil, err
	}
	return &MetricsV1beta1Client{client}, nil
}


// NewForConfigOrDie creates a new MetricsV1beta1Client for the given config and
// panics if there is an error in the config.
func NewForConfigOrDie(c *rest.Config) *MetricsV1beta1Client {
	client, err := NewForConfig(c)
	if err != nil {
		panic(err)
	}
	return client
}


// New creates a new MetricsV1beta1Client for the given RESTClient.
func New(c rest.Interface) *MetricsV1beta1Client {
	return &MetricsV1beta1Client{c}
}


func setConfigDefaults(config *rest.Config) error {
	gv := v1beta1.SchemeGroupVersion
	config.GroupVersion = &gv
	config.APIPath = "/apis"
	config.NegotiatedSerializer = scheme.Codecs.WithoutConversion()

	if config.UserAgent == "" {
		config.UserAgent = rest.DefaultKubernetesUserAgent()
	}

	return nil
}


// RESTClient returns a RESTClient that is used to communicate
// with API server by this client implementation.
func (c *MetricsV1beta1Client) RESTClient() rest.Interface {
	if c == nil {
		return nil
	}
	return c.restClient
}


// newNodeMetricses returns a NodeMetricses
func newNodeMetricses(c *MetricsV1beta1Client) *nodeMetricses {
	return &nodeMetricses{
		client: c.RESTClient(),
	}
}


// Get takes name of the nodeMetrics, and returns the corresponding nodeMetrics object, and an error if there is any.
func (c *nodeMetricses) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1beta1.NodeMetrics, err error) {
	result = &v1beta1.NodeMetrics{}
	err = c.client.Get().
		Resource("nodes").
		Name(name).
		VersionedParams(&options, scheme.ParameterCodec).
		Do(ctx).
		Into(result)
	return
}


// List takes label and field selectors, and returns the list of NodeMetricses that match those selectors.
func (c *nodeMetricses) List(ctx context.Context, opts v1.ListOptions) (result *v1beta1.NodeMetricsList, err error) {
	var timeout time.Duration
	if opts.TimeoutSeconds != nil {
		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
	}
	result = &v1beta1.NodeMetricsList{}
	err = c.client.Get().
		Resource("nodes").
		VersionedParams(&opts, scheme.ParameterCodec).
		Timeout(timeout).
		Do(ctx).
		Into(result)
	return
}


// newPodMetricses returns a PodMetricses
func newPodMetricses(c *MetricsV1beta1Client, namespace string) *podMetricses {
	return &podMetricses{
		client: c.RESTClient(),
		ns:     namespace,
	}
}


// Get takes name of the podMetrics, and returns the corresponding podMetrics object, and an error if there is any.
func (c *podMetricses) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1beta1.PodMetrics, err error) {
	result = &v1beta1.PodMetrics{}
	err = c.client.Get().
		Namespace(c.ns).
		Resource("pods").
		Name(name).
		VersionedParams(&options, scheme.ParameterCodec).
		Do(ctx).
		Into(result)
	return
}


// List takes label and field selectors, and returns the list of PodMetricses that match those selectors.
func (c *podMetricses) List(ctx context.Context, opts v1.ListOptions) (result *v1beta1.PodMetricsList, err error) {
	var timeout time.Duration
	if opts.TimeoutSeconds != nil {
		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
	}
	result = &v1beta1.PodMetricsList{}
	err = c.client.Get().
		Namespace(c.ns).
		Resource("pods").
		VersionedParams(&opts, scheme.ParameterCodec).
		Timeout(timeout).
		Do(ctx).
		Into(result)
	return
}


// NewMetricConverter creates a MetricConverter which knows how to convert objects
// between different versions of the custom metrics api.
func NewMetricConverter() *MetricConverter {
	return &MetricConverter{
		scheme: scheme.Scheme,
		codecs: serializer.NewCodecFactory(scheme.Scheme),
		internalVersioner: runtime.NewMultiGroupVersioner(
			scheme.SchemeGroupVersion,
			schema.GroupKind{Group: cmint.GroupName, Kind: ""},
			schema.GroupKind{Group: cmv1beta1.GroupName, Kind: ""},
			schema.GroupKind{Group: cmv1beta2.GroupName, Kind: ""},
		),
	}
}


// Scheme returns the scheme used by this metric converter.
func (c *MetricConverter) Scheme() *runtime.Scheme {
	return c.scheme
}


// Codecs returns the codecs used by this metric converter
func (c *MetricConverter) Codecs() serializer.CodecFactory {
	return c.codecs
}


// ConvertListOptionsToVersion converts converts a set of MetricListOptions
// to the provided GroupVersion.
func (c *MetricConverter) ConvertListOptionsToVersion(opts *cmint.MetricListOptions, version schema.GroupVersion) (runtime.Object, error) {
	paramObj, err := c.UnsafeConvertToVersionVia(opts, version)
	if err != nil {
		return nil, err
	}
	return paramObj, nil
}


// ConvertResultToVersion converts a Result to the provided GroupVersion
func (c *MetricConverter) ConvertResultToVersion(res rest.Result, gv schema.GroupVersion) (runtime.Object, error) {
	if err := res.Error(); err != nil {
		return nil, err
	}

	metricBytes, err := res.Raw()
	if err != nil {
		return nil, err
	}

	decoder := c.codecs.UniversalDecoder(MetricVersions...)
	rawMetricObj, err := runtime.Decode(decoder, metricBytes)
	if err != nil {
		return nil, err
	}

	metricObj, err := c.UnsafeConvertToVersionVia(rawMetricObj, gv)
	if err != nil {
		return nil, err
	}
	return metricObj, nil
}


// UnsafeConvertToVersionVia is like Scheme.UnsafeConvertToVersion, but it does so via an internal version first.
// We use it here to work with the v1beta2 client internally, while preserving backwards compatibility for existing custom metrics adapters
func (c *MetricConverter) UnsafeConvertToVersionVia(obj runtime.Object, externalVersion schema.GroupVersion) (runtime.Object, error) {
	objInt, err := c.scheme.UnsafeConvertToVersion(obj, schema.GroupVersion{Group: externalVersion.Group, Version: runtime.APIVersionInternal})
	if err != nil {
		return nil, fmt.Errorf("failed to convert the given object to the internal version: %v", err)
	}

	objExt, err := c.scheme.UnsafeConvertToVersion(objInt, externalVersion)
	if err != nil {
		return nil, fmt.Errorf("failed to convert the given object back to the external version: %v", err)
	}

	return objExt, err
}


func init() {
	metricVersionsToGV = make(map[string]schema.GroupVersion)
	for _, ver := range MetricVersions {
		metricVersionsToGV[ver.String()] = ver
	}
}


// NewAvailableAPIsGetter creates an AvailableAPIsGetter that checks discovery
// to find the available versions of the custom metrics api.
func NewAvailableAPIsGetter(client discovery.DiscoveryInterface) AvailableAPIsGetter {
	return &apiVersionsFromDiscovery{
		client: client,
	}
}


// fetchVersions fetches the versions, but doesn't try to invalidate on cache misses.
func (d *apiVersionsFromDiscovery) fetchVersions() (*metav1.APIGroup, error) {
	// TODO(directxman12): amend the discovery interface to ask for a particular group (/apis/foo)
	groups, err := d.client.ServerGroups()
	if err != nil {
		return nil, err
	}

	// Determine the preferred version on the server by first finding the custom metrics group
	var apiGroup *metav1.APIGroup
	for _, group := range groups.Groups {
		if group.Name == cmint.GroupName {
			apiGroup = &group
			break
		}
	}

	if apiGroup == nil {
		return nil, fmt.Errorf("no custom metrics API (%s) registered", cmint.GroupName)
	}

	return apiGroup, nil
}


// chooseVersion sets a preferred version of the custom metrics api based on available versions.
func (d *apiVersionsFromDiscovery) chooseVersion(apiGroup *metav1.APIGroup) (schema.GroupVersion, error) {
	var preferredVersion *schema.GroupVersion
	if gv, present := metricVersionsToGV[apiGroup.PreferredVersion.GroupVersion]; present && len(apiGroup.PreferredVersion.GroupVersion) != 0 {
		preferredVersion = &gv
	} else {
		for _, version := range apiGroup.Versions {
			if gv, present := metricVersionsToGV[version.GroupVersion]; present {
				preferredVersion = &gv
				break
			}
		}
	}

	if preferredVersion == nil {
		return schema.GroupVersion{}, fmt.Errorf("no known available metric versions found")
	}
	return *preferredVersion, nil
}


// PreferredVersion returns the current preferred version of the custom metrics api.
// If none is specified, it will use the first known one.
func (d *apiVersionsFromDiscovery) PreferredVersion() (schema.GroupVersion, error) {
	d.mu.RLock()
	if d.prefVersion != nil {
		// if we've already got one, proceed with that
		defer d.mu.RUnlock()
		return *d.prefVersion, nil
	}
	d.mu.RUnlock()

	d.mu.Lock()
	defer d.mu.Unlock()

	// double check, someone might have beaten us to it
	if d.prefVersion != nil {
		return *d.prefVersion, nil
	}

	// populate our cache
	groupInfo, err := d.fetchVersions()
	if err != nil {
		return schema.GroupVersion{}, err
	}
	prefVersion, err := d.chooseVersion(groupInfo)
	if err != nil {
		return schema.GroupVersion{}, err
	}

	d.prefVersion = &prefVersion
	return *d.prefVersion, nil
}


// Invalidate refreshes the preferred version information.
func (d *apiVersionsFromDiscovery) Invalidate() {
	d.mu.Lock()
	defer d.mu.Unlock()

	d.prefVersion = nil
}


// PeriodicallyInvalidate periodically invalidates the preferred version cache until
// told to stop.
func PeriodicallyInvalidate(cache AvailableAPIsGetter, interval time.Duration, stopCh <-chan struct{}) {
	ticker := time.NewTicker(interval)
	defer ticker.Stop()

	for {
		select {
		case <-ticker.C:
			cache.Invalidate()
		case <-stopCh:
			return
		}
	}
}


// NewForConfig creates a new custom metrics client which delegates to a client which
// uses the preferred api version.
func NewForConfig(baseConfig *rest.Config, mapper meta.RESTMapper, availableAPIs AvailableAPIsGetter) CustomMetricsClient {
	return &multiClient{
		clients:       make(map[schema.GroupVersion]CustomMetricsClient),
		availableAPIs: availableAPIs,

		newClient: func(ver schema.GroupVersion) (CustomMetricsClient, error) {
			return NewForVersionForConfig(rest.CopyConfig(baseConfig), mapper, ver)
		},
	}
}


// getPreferredClient returns a custom metrics client of the preferred api version.
func (c *multiClient) getPreferredClient() (CustomMetricsClient, error) {
	pref, err := c.availableAPIs.PreferredVersion()
	if err != nil {
		return nil, err
	}

	c.mu.RLock()
	client, present := c.clients[pref]
	c.mu.RUnlock()
	if present {
		return client, nil
	}

	c.mu.Lock()
	defer c.mu.Unlock()
	client, err = c.newClient(pref)
	if err != nil {
		return nil, err
	}
	c.clients[pref] = client

	return client, nil
}


func (c *multiClient) RootScopedMetrics() MetricsInterface {
	return &multiClientInterface{clients: c}
}


func (c *multiClient) NamespacedMetrics(namespace string) MetricsInterface {
	return &multiClientInterface{
		clients:   c,
		namespace: &namespace,
	}
}


func (m *multiClientInterface) GetForObject(groupKind schema.GroupKind, name string, metricName string, metricSelector labels.Selector) (*v1beta2.MetricValue, error) {
	client, err := m.clients.getPreferredClient()
	if err != nil {
		return nil, err
	}
	if m.namespace == nil {
		return client.RootScopedMetrics().GetForObject(groupKind, name, metricName, metricSelector)
	} else {
		return client.NamespacedMetrics(*m.namespace).GetForObject(groupKind, name, metricName, metricSelector)
	}
}


func (m *multiClientInterface) GetForObjects(groupKind schema.GroupKind, selector labels.Selector, metricName string, metricSelector labels.Selector) (*v1beta2.MetricValueList, error) {
	client, err := m.clients.getPreferredClient()
	if err != nil {
		return nil, err
	}
	if m.namespace == nil {
		return client.RootScopedMetrics().GetForObjects(groupKind, selector, metricName, metricSelector)
	} else {
		return client.NamespacedMetrics(*m.namespace).GetForObjects(groupKind, selector, metricName, metricSelector)
	}
}


func init() {
	metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"})
	AddToScheme(Scheme)
}


// AddToScheme adds all types of this clientset into the given scheme. This allows composition
// of clientsets, like in:
//
//	import (
//	  "k8s.io/client-go/kubernetes"
//	  clientsetscheme "k8s.io/client-go/kubernetes/scheme"
//	  aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme"
//	)
//
//	kclientset, _ := kubernetes.NewForConfig(c)
//	aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme)
//
// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types
// correctly.
func AddToScheme(scheme *runtime.Scheme) {
	cmint.AddToScheme(scheme)
	cmv1beta1.AddToScheme(scheme)
	cmv1beta2.AddToScheme(scheme)
}


// NewForVersion returns a new CustomMetricsClient for a particular api version.
func NewForVersion(client rest.Interface, mapper meta.RESTMapper, version schema.GroupVersion) CustomMetricsClient {
	return &customMetricsClient{
		client:  client,
		version: version,
		mapper:  mapper,
	}
}


// NewForVersionForConfig returns a new CustomMetricsClient for a particular api version and base configuration.
func NewForVersionForConfig(c *rest.Config, mapper meta.RESTMapper, version schema.GroupVersion) (CustomMetricsClient, error) {
	configShallowCopy := *c
	if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 {
		if configShallowCopy.Burst <= 0 {
			return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0")
		}
		configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst)
	}
	configShallowCopy.APIPath = "/apis"
	if configShallowCopy.UserAgent == "" {
		configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent()
	}
	configShallowCopy.GroupVersion = &version
	configShallowCopy.NegotiatedSerializer = scheme.Codecs.WithoutConversion()

	client, err := rest.RESTClientFor(&configShallowCopy)
	if err != nil {
		return nil, err
	}

	return NewForVersion(client, mapper, version), nil
}


func (c *customMetricsClient) RootScopedMetrics() MetricsInterface {
	return &rootScopedMetrics{c}
}


func (c *customMetricsClient) NamespacedMetrics(namespace string) MetricsInterface {
	return &namespacedMetrics{
		client:    c,
		namespace: namespace,
	}
}


// qualResourceForKind returns the string format of a qualified group resource for the specified GroupKind
func (c *customMetricsClient) qualResourceForKind(groupKind schema.GroupKind) (string, error) {
	mapping, err := c.mapper.RESTMapping(groupKind)
	if err != nil {
		return "", fmt.Errorf("unable to map kind %s to resource: %v", groupKind.String(), err)
	}

	gr := mapping.Resource.GroupResource()
	return gr.String(), nil
}