func MapBlockVolume(

// MapBlockVolume is a utility function to provide a common way of mapping
// block device path for a specified volume and pod.  This function should be
// called by volume plugins that implements volume.BlockVolumeMapper.Map() method.
func MapBlockVolume(
	blkUtil volumepathhandler.BlockVolumePathHandler,
	devicePath,
	globalMapPath,
	podVolumeMapPath,
	volumeMapName string,
	podUID utypes.UID,
) error {
	// map devicePath to global node path as bind mount
	mapErr := blkUtil.MapDevice(devicePath, globalMapPath, string(podUID), true /* bindMount */)
	if mapErr != nil {
		return fmt.Errorf("blkUtil.MapDevice failed. devicePath: %s, globalMapPath:%s, podUID: %s, bindMount: %v: %v",
			devicePath, globalMapPath, string(podUID), true, mapErr)
	}

	// map devicePath to pod volume path
	mapErr = blkUtil.MapDevice(devicePath, podVolumeMapPath, volumeMapName, false /* bindMount */)
	if mapErr != nil {
		return fmt.Errorf("blkUtil.MapDevice failed. devicePath: %s, podVolumeMapPath:%s, volumeMapName: %s, bindMount: %v: %v",
			devicePath, podVolumeMapPath, volumeMapName, false, mapErr)
	}

	// Take file descriptor lock to keep a block device opened. Otherwise, there is a case
	// that the block device is silently removed and attached another device with the same name.
	// Container runtime can't handle this problem. To avoid unexpected condition fd lock
	// for the block device is required.
	_, mapErr = blkUtil.AttachFileDevice(filepath.Join(globalMapPath, string(podUID)))
	if mapErr != nil {
		return fmt.Errorf("blkUtil.AttachFileDevice failed. globalMapPath:%s, podUID: %s: %v",
			globalMapPath, string(podUID), mapErr)
	}

	return nil
}


func UnmapBlockVolume(

// UnmapBlockVolume is a utility function to provide a common way of unmapping
// block device path for a specified volume and pod.  This function should be
// called by volume plugins that implements volume.BlockVolumeMapper.Map() method.
func UnmapBlockVolume(
	blkUtil volumepathhandler.BlockVolumePathHandler,
	globalUnmapPath,
	podDeviceUnmapPath,
	volumeMapName string,
	podUID utypes.UID,
) error {
	// Release file descriptor lock.
	err := blkUtil.DetachFileDevice(filepath.Join(globalUnmapPath, string(podUID)))
	if err != nil {
		return fmt.Errorf("blkUtil.DetachFileDevice failed. globalUnmapPath:%s, podUID: %s: %v",
			globalUnmapPath, string(podUID), err)
	}

	// unmap devicePath from pod volume path
	unmapDeviceErr := blkUtil.UnmapDevice(podDeviceUnmapPath, volumeMapName, false /* bindMount */)
	if unmapDeviceErr != nil {
		return fmt.Errorf("blkUtil.DetachFileDevice failed. podDeviceUnmapPath:%s, volumeMapName: %s, bindMount: %v: %v",
			podDeviceUnmapPath, volumeMapName, false, unmapDeviceErr)
	}

	// unmap devicePath from global node path
	unmapDeviceErr = blkUtil.UnmapDevice(globalUnmapPath, string(podUID), true /* bindMount */)
	if unmapDeviceErr != nil {
		return fmt.Errorf("blkUtil.DetachFileDevice failed. globalUnmapPath:%s, podUID: %s, bindMount: %v: %v",
			globalUnmapPath, string(podUID), true, unmapDeviceErr)
	}
	return nil
}


// GetPluginMountDir returns the global mount directory name appended
// to the given plugin name's plugin directory
func GetPluginMountDir(host volume.VolumeHost, name string) string {
	mntDir := filepath.Join(host.GetPluginDir(name), MountsInGlobalPDPath)
	return mntDir
}


// IsLocalEphemeralVolume determines whether the argument is a local ephemeral
// volume vs. some other type
// Local means the volume is using storage from the local disk that is managed by kubelet.
// Ephemeral means the lifecycle of the volume is the same as the Pod.
func IsLocalEphemeralVolume(volume v1.Volume) bool {
	return volume.GitRepo != nil ||
		(volume.EmptyDir != nil && volume.EmptyDir.Medium == v1.StorageMediumDefault) ||
		volume.ConfigMap != nil
}


// GetLocalPersistentVolumeNodeNames returns the node affinity node name(s) for
// local PersistentVolumes. nil is returned if the PV does not have any
// specific node affinity node selector terms and match expressions.
// PersistentVolume with node affinity has select and match expressions
// in the form of:
//
//	nodeAffinity:
//	  required:
//	    nodeSelectorTerms:
//	    - matchExpressions:
//	      - key: kubernetes.io/hostname
//	        operator: In
//	        values:
//	        - <node1>
//	        - <node2>
func GetLocalPersistentVolumeNodeNames(pv *v1.PersistentVolume) []string {
	if pv == nil || pv.Spec.NodeAffinity == nil || pv.Spec.NodeAffinity.Required == nil {
		return nil
	}

	var result sets.Set[string]
	for _, term := range pv.Spec.NodeAffinity.Required.NodeSelectorTerms {
		var nodes sets.Set[string]
		for _, matchExpr := range term.MatchExpressions {
			if matchExpr.Key == v1.LabelHostname && matchExpr.Operator == v1.NodeSelectorOpIn {
				if nodes == nil {
					nodes = sets.New(matchExpr.Values...)
				} else {
					nodes = nodes.Intersection(sets.New(matchExpr.Values...))
				}
			}
		}
		result = result.Union(nodes)
	}

	return sets.List(result)
}


// GetPodVolumeNames returns names of volumes that are used in a pod,
// either as filesystem mount or raw block device, together with list
// of all SELinux contexts of all containers that use the volumes.
func GetPodVolumeNames(pod *v1.Pod) (mounts sets.String, devices sets.String, seLinuxContainerContexts map[string][]*v1.SELinuxOptions) {
	mounts = sets.NewString()
	devices = sets.NewString()
	seLinuxContainerContexts = make(map[string][]*v1.SELinuxOptions)

	podutil.VisitContainers(&pod.Spec, podutil.AllFeatureEnabledContainers(), func(container *v1.Container, containerType podutil.ContainerType) bool {
		var seLinuxOptions *v1.SELinuxOptions
		if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) {
			effectiveContainerSecurity := securitycontext.DetermineEffectiveSecurityContext(pod, container)
			if effectiveContainerSecurity != nil {
				// No DeepCopy, SELinuxOptions is already a copy of Pod's or container's SELinuxOptions
				seLinuxOptions = effectiveContainerSecurity.SELinuxOptions
			}
		}

		if container.VolumeMounts != nil {
			for _, mount := range container.VolumeMounts {
				mounts.Insert(mount.Name)
				if seLinuxOptions != nil {
					seLinuxContainerContexts[mount.Name] = append(seLinuxContainerContexts[mount.Name], seLinuxOptions.DeepCopy())
				}
			}
		}
		if container.VolumeDevices != nil {
			for _, device := range container.VolumeDevices {
				devices.Insert(device.Name)
			}
		}
		return true
	})
	return
}


func FsUserFrom(pod *v1.Pod) *int64

// FsUserFrom returns FsUser of pod, which is determined by the runAsUser
// attributes.
func FsUserFrom(pod *v1.Pod) *int64 {
	var fsUser *int64
	// Exclude ephemeral containers because SecurityContext is not allowed.
	podutil.VisitContainers(&pod.Spec, podutil.InitContainers|podutil.Containers, func(container *v1.Container, containerType podutil.ContainerType) bool {
		runAsUser, ok := securitycontext.DetermineEffectiveRunAsUser(pod, container)
		// One container doesn't specify user or there are more than one
		// non-root UIDs.
		if !ok || (fsUser != nil && *fsUser != *runAsUser) {
			fsUser = nil
			return false
		}
		if fsUser == nil {
			fsUser = runAsUser
		}
		return true
	})
	return fsUser
}


// HasMountRefs checks if the given mountPath has mountRefs.
// TODO: this is a workaround for the unmount device issue caused by gci mounter.
// In GCI cluster, if gci mounter is used for mounting, the container started by mounter
// script will cause additional mounts created in the container. Since these mounts are
// irrelevant to the original mounts, they should be not considered when checking the
// mount references. The current solution is to filter out those mount paths that contain
// the k8s plugin suffix of original mount path.
func HasMountRefs(mountPath string, mountRefs []string) bool {
	// A mountPath typically is like
	//   /var/lib/kubelet/plugins/kubernetes.io/some-plugin/mounts/volume-XXXX
	// Mount refs can look like
	//   /home/somewhere/var/lib/kubelet/plugins/kubernetes.io/some-plugin/...
	// but if /var/lib/kubelet is mounted to a different device a ref might be like
	//   /mnt/some-other-place/kubelet/plugins/kubernetes.io/some-plugin/...
	// Neither of the above should be counted as a mount ref as those are handled
	// by the kubelet. What we're concerned about is a path like
	//   /data/local/some/manual/mount
	// As unmonting could interrupt usage from that mountpoint.
	//
	// So instead of looking for the entire /var/lib/... path, the plugins/kuberentes.io/
	// suffix is trimmed off and searched for.
	//
	// If there isn't a /plugins/... path, the whole mountPath is used instead.
	pathToFind := mountPath
	if i := strings.Index(mountPath, kubernetesPluginPathPrefix); i > -1 {
		pathToFind = mountPath[i:]
	}
	for _, ref := range mountRefs {
		if !strings.Contains(ref, pathToFind) {
			return true
		}
	}
	return false
}


// WriteVolumeCache flush disk data given the spcified mount path
func WriteVolumeCache(deviceMountPath string, exec utilexec.Interface) error {
	// If runtime os is windows, execute Write-VolumeCache powershell command on the disk
	if runtime.GOOS == "windows" {
		cmd := fmt.Sprintf("Get-Volume -FilePath %s | Write-Volumecache", deviceMountPath)
		output, err := exec.Command("powershell", "/c", cmd).CombinedOutput()
		klog.Infof("command (%q) execeuted: %v, output: %q", cmd, err, string(output))
		if err != nil {
			return fmt.Errorf("command (%q) failed: %v, output: %q", cmd, err, string(output))
		}
	}
	// For linux runtime, it skips because unmount will automatically flush disk data
	return nil
}


// IsMultiAttachAllowed checks if attaching this volume to multiple nodes is definitely not allowed/possible.
// In its current form, this function can only reliably say for which volumes it's definitely forbidden. If it returns
// false, it is not guaranteed that multi-attach is actually supported by the volume type and we must rely on the
// attacher to fail fast in such cases.
// Please see https://github.com/kubernetes/kubernetes/issues/40669 and https://github.com/kubernetes/kubernetes/pull/40148#discussion_r98055047
func IsMultiAttachAllowed(volumeSpec *volume.Spec) bool {
	if volumeSpec == nil {
		// we don't know if it's supported or not and let the attacher fail later in cases it's not supported
		return true
	}

	if volumeSpec.Volume != nil {
		// Check for volume types which are known to fail slow or cause trouble when trying to multi-attach
		if volumeSpec.Volume.AzureDisk != nil ||
			volumeSpec.Volume.Cinder != nil {
			return false
		}
	}

	// Only if this volume is a persistent volume, we have reliable information on whether it's allowed or not to
	// multi-attach. We trust in the individual volume implementations to not allow unsupported access modes
	if volumeSpec.PersistentVolume != nil {
		// Check for persistent volume types which do not fail when trying to multi-attach
		if len(volumeSpec.PersistentVolume.Spec.AccessModes) == 0 {
			// No access mode specified so we don't know for sure. Let the attacher fail if needed
			return true
		}

		// check if this volume is allowed to be attached to multiple PODs/nodes, if yes, return false
		for _, accessMode := range volumeSpec.PersistentVolume.Spec.AccessModes {
			if accessMode == v1.ReadWriteMany || accessMode == v1.ReadOnlyMany {
				return true
			}
		}
		return false
	}

	// we don't know if it's supported or not and let the attacher fail later in cases it's not supported
	return true
}


// IsAttachableVolume checks if the given volumeSpec is an attachable volume or not
func IsAttachableVolume(volumeSpec *volume.Spec, volumePluginMgr *volume.VolumePluginMgr) bool {
	attachableVolumePlugin, _ := volumePluginMgr.FindAttachablePluginBySpec(volumeSpec)
	if attachableVolumePlugin != nil {
		volumeAttacher, err := attachableVolumePlugin.NewAttacher()
		if err == nil && volumeAttacher != nil {
			return true
		}
	}

	return false
}


// IsDeviceMountableVolume checks if the given volumeSpec is an device mountable volume or not
func IsDeviceMountableVolume(volumeSpec *volume.Spec, volumePluginMgr *volume.VolumePluginMgr) bool {
	deviceMountableVolumePlugin, _ := volumePluginMgr.FindDeviceMountablePluginBySpec(volumeSpec)
	if deviceMountableVolumePlugin != nil {
		volumeDeviceMounter, err := deviceMountableVolumePlugin.NewDeviceMounter()
		if err == nil && volumeDeviceMounter != nil {
			return true
		}
	}

	return false
}


// GetReliableMountRefs calls mounter.GetMountRefs and retries on IsInconsistentReadError.
// To be used in volume reconstruction of volume plugins that don't have any protection
// against mounting a single volume on multiple nodes (such as attach/detach).
func GetReliableMountRefs(mounter mount.Interface, mountPath string) ([]string, error) {
	var paths []string
	var lastErr error
	err := wait.PollImmediate(10*time.Millisecond, time.Minute, func() (bool, error) {
		var err error
		paths, err = mounter.GetMountRefs(mountPath)
		if io.IsInconsistentReadError(err) {
			lastErr = err
			return false, nil
		}
		if err != nil {
			return false, err
		}
		return true, nil
	})
	if err == wait.ErrWaitTimeout {
		return nil, lastErr
	}
	return paths, err
}


// NewBlockVolumePathHandler returns a new instance of BlockVolumeHandler.
func NewBlockVolumePathHandler() BlockVolumePathHandler {
	var volumePathHandler VolumePathHandler
	return volumePathHandler
}


// MapDevice creates a symbolic link to block device under specified map path
func (v VolumePathHandler) MapDevice(devicePath string, mapPath string, linkName string, bindMount bool) error {
	// Example of global map path:
	//   globalMapPath/linkName: plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{podUid}
	//   linkName: {podUid}
	//
	// Example of pod device map path:
	//   podDeviceMapPath/linkName: pods/{podUid}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName}
	//   linkName: {volumeName}
	if len(devicePath) == 0 {
		return fmt.Errorf("failed to map device to map path. devicePath is empty")
	}
	if len(mapPath) == 0 {
		return fmt.Errorf("failed to map device to map path. mapPath is empty")
	}
	if !filepath.IsAbs(mapPath) {
		return fmt.Errorf("the map path should be absolute: map path: %s", mapPath)
	}
	klog.V(5).Infof("MapDevice: devicePath %s", devicePath)
	klog.V(5).Infof("MapDevice: mapPath %s", mapPath)
	klog.V(5).Infof("MapDevice: linkName %s", linkName)

	// Check and create mapPath
	_, err := os.Stat(mapPath)
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("cannot validate map path: %s: %v", mapPath, err)
	}
	if err = os.MkdirAll(mapPath, 0750); err != nil {
		return fmt.Errorf("failed to mkdir %s: %v", mapPath, err)
	}

	if bindMount {
		return mapBindMountDevice(v, devicePath, mapPath, linkName)
	}
	return mapSymlinkDevice(v, devicePath, mapPath, linkName)
}


func mapBindMountDevice(v VolumePathHandler, devicePath string, mapPath string, linkName string) error {
	// Check bind mount exists
	linkPath := filepath.Join(mapPath, string(linkName))

	file, err := os.Stat(linkPath)
	if err != nil {
		if !os.IsNotExist(err) {
			return fmt.Errorf("failed to stat file %s: %v", linkPath, err)
		}

		// Create file
		newFile, err := os.OpenFile(linkPath, os.O_CREATE|os.O_RDWR, 0750)
		if err != nil {
			return fmt.Errorf("failed to open file %s: %v", linkPath, err)
		}
		if err := newFile.Close(); err != nil {
			return fmt.Errorf("failed to close file %s: %v", linkPath, err)
		}
	} else {
		// Check if device file
		// TODO: Need to check if this device file is actually the expected bind mount
		if file.Mode()&os.ModeDevice == os.ModeDevice {
			klog.Warningf("Warning: Map skipped because bind mount already exist on the path: %v", linkPath)
			return nil
		}

		klog.Warningf("Warning: file %s is already exist but not mounted, skip creating file", linkPath)
	}

	// Bind mount file
	mounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: utilexec.New()}
	if err := mounter.MountSensitiveWithoutSystemd(devicePath, linkPath, "" /* fsType */, []string{"bind"}, nil); err != nil {
		return fmt.Errorf("failed to bind mount devicePath: %s to linkPath %s: %v", devicePath, linkPath, err)
	}

	return nil
}


func mapSymlinkDevice(v VolumePathHandler, devicePath string, mapPath string, linkName string) error {
	// Remove old symbolic link(or file) then create new one.
	// This should be done because current symbolic link is
	// stale across node reboot.
	linkPath := filepath.Join(mapPath, string(linkName))
	if err := os.Remove(linkPath); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("failed to remove file %s: %v", linkPath, err)
	}
	return os.Symlink(devicePath, linkPath)
}


// UnmapDevice removes a symbolic link associated to block device under specified map path
func (v VolumePathHandler) UnmapDevice(mapPath string, linkName string, bindMount bool) error {
	if len(mapPath) == 0 {
		return fmt.Errorf("failed to unmap device from map path. mapPath is empty")
	}
	klog.V(5).Infof("UnmapDevice: mapPath %s", mapPath)
	klog.V(5).Infof("UnmapDevice: linkName %s", linkName)

	if bindMount {
		return unmapBindMountDevice(v, mapPath, linkName)
	}
	return unmapSymlinkDevice(v, mapPath, linkName)
}


func unmapBindMountDevice(v VolumePathHandler, mapPath string, linkName string) error {
	// Check bind mount exists
	linkPath := filepath.Join(mapPath, string(linkName))
	if isMountExist, checkErr := v.IsDeviceBindMountExist(linkPath); checkErr != nil {
		return checkErr
	} else if !isMountExist {
		klog.Warningf("Warning: Unmap skipped because bind mount does not exist on the path: %v", linkPath)

		// Check if linkPath still exists
		if _, err := os.Stat(linkPath); err != nil {
			if !os.IsNotExist(err) {
				return fmt.Errorf("failed to check if path %s exists: %v", linkPath, err)
			}
			// linkPath has already been removed
			return nil
		}
		// Remove file
		if err := os.Remove(linkPath); err != nil && !os.IsNotExist(err) {
			return fmt.Errorf("failed to remove file %s: %v", linkPath, err)
		}
		return nil
	}

	// Unmount file
	mounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: utilexec.New()}
	if err := mounter.Unmount(linkPath); err != nil {
		return fmt.Errorf("failed to unmount linkPath %s: %v", linkPath, err)
	}

	// Remove file
	if err := os.Remove(linkPath); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("failed to remove file %s: %v", linkPath, err)
	}

	return nil
}


func unmapSymlinkDevice(v VolumePathHandler, mapPath string, linkName string) error {
	// Check symbolic link exists
	linkPath := filepath.Join(mapPath, string(linkName))
	if islinkExist, checkErr := v.IsSymlinkExist(linkPath); checkErr != nil {
		return checkErr
	} else if !islinkExist {
		klog.Warningf("Warning: Unmap skipped because symlink does not exist on the path: %v", linkPath)
		return nil
	}
	return os.Remove(linkPath)
}


// RemoveMapPath removes a file or directory on specified map path
func (v VolumePathHandler) RemoveMapPath(mapPath string) error {
	if len(mapPath) == 0 {
		return fmt.Errorf("failed to remove map path. mapPath is empty")
	}
	klog.V(5).Infof("RemoveMapPath: mapPath %s", mapPath)
	err := os.RemoveAll(mapPath)
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("failed to remove directory %s: %v", mapPath, err)
	}
	return nil
}


// IsSymlinkExist returns true if specified file exists and the type is symbolik link.
// If file doesn't exist, or file exists but not symbolic link, return false with no error.
// On other cases, return false with error from Lstat().
func (v VolumePathHandler) IsSymlinkExist(mapPath string) (bool, error) {
	fi, err := os.Lstat(mapPath)
	if err != nil {
		// If file doesn't exist, return false and no error
		if os.IsNotExist(err) {
			return false, nil
		}
		// Return error from Lstat()
		return false, fmt.Errorf("failed to Lstat file %s: %v", mapPath, err)
	}
	// If file exits and it's symbolic link, return true and no error
	if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
		return true, nil
	}
	// If file exits but it's not symbolic link, return false and no error
	return false, nil
}


// IsDeviceBindMountExist returns true if specified file exists and the type is device.
// If file doesn't exist, or file exists but not device, return false with no error.
// On other cases, return false with error from Lstat().
func (v VolumePathHandler) IsDeviceBindMountExist(mapPath string) (bool, error) {
	fi, err := os.Lstat(mapPath)
	if err != nil {
		// If file doesn't exist, return false and no error
		if os.IsNotExist(err) {
			return false, nil
		}

		// Return error from Lstat()
		return false, fmt.Errorf("failed to Lstat file %s: %v", mapPath, err)
	}
	// If file exits and it's device, return true and no error
	if fi.Mode()&os.ModeDevice == os.ModeDevice {
		return true, nil
	}
	// If file exits but it's not device, return false and no error
	return false, nil
}


// GetDeviceBindMountRefs searches bind mounts under global map path
func (v VolumePathHandler) GetDeviceBindMountRefs(devPath string, mapPath string) ([]string, error) {
	var refs []string
	files, err := ioutil.ReadDir(mapPath)
	if err != nil {
		return nil, err
	}
	for _, file := range files {
		if file.Mode()&os.ModeDevice != os.ModeDevice {
			continue
		}
		filename := file.Name()
		// TODO: Might need to check if the file is actually linked to devPath
		refs = append(refs, filepath.Join(mapPath, filename))
	}
	klog.V(5).Infof("GetDeviceBindMountRefs: refs %v", refs)
	return refs, nil
}


// AttachFileDevice takes a path to a regular file and makes it available as an
// attached block device.
func (v VolumePathHandler) AttachFileDevice(path string) (string, error) {
	blockDevicePath, err := v.GetLoopDevice(path)
	if err != nil && err.Error() != ErrDeviceNotFound {
		return "", fmt.Errorf("GetLoopDevice failed for path %s: %v", path, err)
	}

	// If no existing loop device for the path, create one
	if blockDevicePath == "" {
		klog.V(4).Infof("Creating device for path: %s", path)
		blockDevicePath, err = makeLoopDevice(path)
		if err != nil {
			return "", fmt.Errorf("makeLoopDevice failed for path %s: %v", path, err)
		}
	}
	return blockDevicePath, nil
}


// DetachFileDevice takes a path to the attached block device and
// detach it from block device.
func (v VolumePathHandler) DetachFileDevice(path string) error {
	loopPath, err := v.GetLoopDevice(path)
	if err != nil {
		if err.Error() == ErrDeviceNotFound {
			klog.Warningf("couldn't find loopback device which takes file descriptor lock. Skip detaching device. device path: %q", path)
		} else {
			return fmt.Errorf("GetLoopDevice failed for path %s: %v", path, err)
		}
	} else {
		if len(loopPath) != 0 {
			err = removeLoopDevice(loopPath)
			if err != nil {
				return fmt.Errorf("removeLoopDevice failed for path %s: %v", path, err)
			}
		}
	}
	return nil
}


// GetLoopDevice returns the full path to the loop device associated with the given path.
func (v VolumePathHandler) GetLoopDevice(path string) (string, error) {
	_, err := os.Stat(path)
	if os.IsNotExist(err) {
		return "", errors.New(ErrDeviceNotFound)
	}
	if err != nil {
		return "", fmt.Errorf("not attachable: %v", err)
	}

	return getLoopDeviceFromSysfs(path)
}


func makeLoopDevice(path string) (string, error) {
	args := []string{"-f", path}
	cmd := exec.Command(losetupPath, args...)

	out, err := cmd.CombinedOutput()
	if err != nil {
		klog.V(2).Infof("Failed device create command for path: %s %v %s", path, err, out)
		return "", fmt.Errorf("losetup %s failed: %v", strings.Join(args, " "), err)
	}

	return getLoopDeviceFromSysfs(path)
}


// removeLoopDevice removes specified loopback device
func removeLoopDevice(device string) error {
	args := []string{"-d", device}
	cmd := exec.Command(losetupPath, args...)
	out, err := cmd.CombinedOutput()
	if err != nil {
		if _, err := os.Stat(device); os.IsNotExist(err) {
			return nil
		}
		klog.V(2).Infof("Failed to remove loopback device: %s: %v %s", device, err, out)
		return fmt.Errorf("losetup -d %s failed: %v", device, err)
	}
	return nil
}


// getLoopDeviceFromSysfs finds the backing file for a loop
// device from sysfs via "/sys/block/loop*/loop/backing_file".
func getLoopDeviceFromSysfs(path string) (string, error) {
	// If the file is a symlink.
	realPath, err := filepath.EvalSymlinks(path)
	if err != nil {
		return "", fmt.Errorf("failed to evaluate path %s: %s", path, err)
	}

	devices, err := filepath.Glob("/sys/block/loop*")
	if err != nil {
		return "", fmt.Errorf("failed to list loop devices in sysfs: %s", err)
	}

	for _, device := range devices {
		backingFile := fmt.Sprintf("%s/loop/backing_file", device)

		// The contents of this file is the absolute path of "path".
		data, err := ioutil.ReadFile(backingFile)
		if err != nil {
			continue
		}

		// Return the first match.
		backingFilePath := cleanBackingFilePath(string(data))
		if backingFilePath == path || backingFilePath == realPath {
			return fmt.Sprintf("/dev/%s", filepath.Base(device)), nil
		}
	}

	return "", errors.New(ErrDeviceNotFound)
}


// cleanPath remove any trailing substrings that are not part of the backing file path.
func cleanBackingFilePath(path string) string {
	// If the block device was deleted, the path will contain a "(deleted)" suffix
	path = strings.TrimSpace(path)
	path = strings.TrimSuffix(path, "(deleted)")
	return strings.TrimSpace(path)
}


// FindGlobalMapPathUUIDFromPod finds {pod uuid} bind mount under globalMapPath
// corresponding to map path symlink, and then return global map path with pod uuid.
// (See pkg/volume/volume.go for details on a global map path and a pod device map path.)
// ex. mapPath symlink: pods/{podUid}}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName} -> /dev/sdX
//
//	globalMapPath/{pod uuid} bind mount: plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{pod uuid} -> /dev/sdX
func (v VolumePathHandler) FindGlobalMapPathUUIDFromPod(pluginDir, mapPath string, podUID types.UID) (string, error) {
	var globalMapPathUUID string
	// Find symbolic link named pod uuid under plugin dir
	err := filepath.Walk(pluginDir, func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if (fi.Mode()&os.ModeDevice == os.ModeDevice) && (fi.Name() == string(podUID)) {
			klog.V(5).Infof("FindGlobalMapPathFromPod: path %s, mapPath %s", path, mapPath)
			if res, err := compareBindMountAndSymlinks(path, mapPath); err == nil && res {
				globalMapPathUUID = path
			}
		}
		return nil
	})
	if err != nil {
		return "", fmt.Errorf("FindGlobalMapPathUUIDFromPod failed: %v", err)
	}
	klog.V(5).Infof("FindGlobalMapPathFromPod: globalMapPathUUID %s", globalMapPathUUID)
	// Return path contains global map path + {pod uuid}
	return globalMapPathUUID, nil
}


// compareBindMountAndSymlinks returns if global path (bind mount) and
// pod path (symlink) are pointing to the same device.
// If there is an error in checking it returns error.
func compareBindMountAndSymlinks(global, pod string) (bool, error) {
	// To check if bind mount and symlink are pointing to the same device,
	// we need to check if they are pointing to the devices that have same major/minor number.

	// Get the major/minor number for global path
	devNumGlobal, err := getDeviceMajorMinor(global)
	if err != nil {
		return false, fmt.Errorf("getDeviceMajorMinor failed for path %s: %v", global, err)
	}

	// Get the symlinked device from the pod path
	devPod, err := os.Readlink(pod)
	if err != nil {
		return false, fmt.Errorf("failed to readlink path %s: %v", pod, err)
	}
	// Get the major/minor number for the symlinked device from the pod path
	devNumPod, err := getDeviceMajorMinor(devPod)
	if err != nil {
		return false, fmt.Errorf("getDeviceMajorMinor failed for path %s: %v", devPod, err)
	}
	klog.V(5).Infof("CompareBindMountAndSymlinks: devNumGlobal %s, devNumPod %s", devNumGlobal, devNumPod)

	// Check if the major/minor number are the same
	if devNumGlobal == devNumPod {
		return true, nil
	}
	return false, nil
}


// getDeviceMajorMinor returns major/minor number for the path with below format:
// major:minor (in hex)
// ex)
//
//	fc:10
func getDeviceMajorMinor(path string) (string, error) {
	var stat unix.Stat_t

	if err := unix.Stat(path, &stat); err != nil {
		return "", fmt.Errorf("failed to stat path %s: %v", path, err)
	}

	devNumber := uint64(stat.Rdev)
	major := unix.Major(devNumber)
	minor := unix.Minor(devNumber)

	return fmt.Sprintf("%x:%x", major, minor), nil
}


// AttachFileDevice takes a path to a regular file and makes it available as an
// attached block device.
func (v VolumePathHandler) AttachFileDevice(path string) (string, error) {
	return "", fmt.Errorf("AttachFileDevice not supported for this build.")
}


// DetachFileDevice takes a path to the attached block device and
// detach it from block device.
func (v VolumePathHandler) DetachFileDevice(path string) error {
	return fmt.Errorf("DetachFileDevice not supported for this build.")
}


// GetLoopDevice returns the full path to the loop device associated with the given path.
func (v VolumePathHandler) GetLoopDevice(path string) (string, error) {
	return "", fmt.Errorf("GetLoopDevice not supported for this build.")
}


// FindGlobalMapPathUUIDFromPod finds {pod uuid} bind mount under globalMapPath
// corresponding to map path symlink, and then return global map path with pod uuid.
func (v VolumePathHandler) FindGlobalMapPathUUIDFromPod(pluginDir, mapPath string, podUID types.UID) (string, error) {
	return "", fmt.Errorf("FindGlobalMapPathUUIDFromPod not supported for this build.")
}


// ValidatePersistentVolume validates PV object for plugin specific validation
// We can put here validations which are specific to volume types.
func ValidatePersistentVolume(pv *api.PersistentVolume) field.ErrorList {
	return checkMountOption(pv)
}


func checkMountOption(pv *api.PersistentVolume) field.ErrorList {
	allErrs := field.ErrorList{}
	// if PV is of these types we don't return errors
	// since mount options is supported
	if pv.Spec.GCEPersistentDisk != nil ||
		pv.Spec.AWSElasticBlockStore != nil ||
		pv.Spec.Glusterfs != nil ||
		pv.Spec.NFS != nil ||
		pv.Spec.RBD != nil ||
		pv.Spec.Quobyte != nil ||
		pv.Spec.ISCSI != nil ||
		pv.Spec.Cinder != nil ||
		pv.Spec.CephFS != nil ||
		pv.Spec.AzureFile != nil ||
		pv.Spec.VsphereVolume != nil ||
		pv.Spec.AzureDisk != nil ||
		pv.Spec.PhotonPersistentDisk != nil {
		return allErrs
	}
	// any other type if mount option is present lets return error
	if _, ok := pv.Annotations[api.MountOptionAnnotation]; ok {
		metaField := field.NewPath("metadata")
		allErrs = append(allErrs, field.Forbidden(metaField.Child("annotations", api.MountOptionAnnotation), "may not specify mount options for this volume type"))
	}
	return allErrs
}


// ValidatePathNoBacksteps will make sure the targetPath does not have any element which is ".."
func ValidatePathNoBacksteps(targetPath string) error {
	parts := strings.Split(filepath.ToSlash(targetPath), "/")
	for _, item := range parts {
		if item == ".." {
			return errors.New("must not contain '..'")
		}
	}

	return nil
}


// SetVolumeOwnership modifies the given volume to be owned by
// fsGroup, and sets SetGid so that newly created files are owned by
// fsGroup. If fsGroup is nil nothing is done.
func SetVolumeOwnership(mounter Mounter, dir string, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy, completeFunc func(types.CompleteFuncParam)) error {
	if fsGroup == nil {
		return nil
	}

	timer := time.AfterFunc(30*time.Second, func() {
		klog.Warningf("Setting volume ownership for %s and fsGroup set. If the volume has a lot of files then setting volume ownership could be slow, see https://github.com/kubernetes/kubernetes/issues/69699", dir)
	})
	defer timer.Stop()

	if skipPermissionChange(mounter, dir, fsGroup, fsGroupChangePolicy) {
		klog.V(3).InfoS("Skipping permission and ownership change for volume", "path", dir)
		return nil
	}

	err := walkDeep(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		return changeFilePermission(path, fsGroup, mounter.GetAttributes().ReadOnly, info)
	})
	if completeFunc != nil {
		completeFunc(types.CompleteFuncParam{
			Err: &err,
		})
	}
	return err
}


func changeFilePermission(filename string, fsGroup *int64, readonly bool, info os.FileInfo) error {
	err := os.Lchown(filename, -1, int(*fsGroup))
	if err != nil {
		klog.ErrorS(err, "Lchown failed", "path", filename)
	}

	// chmod passes through to the underlying file for symlinks.
	// Symlinks have a mode of 777 but this really doesn't mean anything.
	// The permissions of the underlying file are what matter.
	// However, if one reads the mode of a symlink then chmods the symlink
	// with that mode, it changes the mode of the underlying file, overridden
	// the defaultMode and permissions initialized by the volume plugin, which
	// is not what we want; thus, we skip chmod for symlinks.
	if info.Mode()&os.ModeSymlink != 0 {
		return nil
	}

	mask := rwMask
	if readonly {
		mask = roMask
	}

	if info.IsDir() {
		mask |= os.ModeSetgid
		mask |= execMask
	}

	err = os.Chmod(filename, info.Mode()|mask)
	if err != nil {
		klog.ErrorS(err, "chmod failed", "path", filename)
	}

	return nil
}


func skipPermissionChange(mounter Mounter, dir string, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy) bool {
	if fsGroupChangePolicy == nil || *fsGroupChangePolicy != v1.FSGroupChangeOnRootMismatch {
		klog.V(4).InfoS("Perform recursive ownership change for directory", "path", dir)
		return false
	}
	return !requiresPermissionChange(dir, fsGroup, mounter.GetAttributes().ReadOnly)
}


func requiresPermissionChange(rootDir string, fsGroup *int64, readonly bool) bool {
	fsInfo, err := os.Stat(rootDir)
	if err != nil {
		klog.ErrorS(err, "Performing recursive ownership change on rootDir because reading permissions of root volume failed", "path", rootDir)
		return true
	}
	stat, ok := fsInfo.Sys().(*syscall.Stat_t)
	if !ok || stat == nil {
		klog.ErrorS(nil, "Performing recursive ownership change on rootDir because reading permissions of root volume failed", "path", rootDir)
		return true
	}

	if int(stat.Gid) != int(*fsGroup) {
		klog.V(4).InfoS("Expected group ownership of volume did not match with Gid", "path", rootDir, "GID", stat.Gid)
		return true
	}
	unixPerms := rwMask

	if readonly {
		unixPerms = roMask
	}

	// if rootDir is not a directory then we should apply permission change anyways
	if !fsInfo.IsDir() {
		return true
	}
	unixPerms |= execMask
	filePerm := fsInfo.Mode().Perm()

	// We need to check if actual permissions of root directory is a superset of permissions required by unixPerms.
	// This is done by checking if permission bits expected in unixPerms is set in actual permissions of the directory.
	// We use bitwise AND operation to check set bits. For example:
	//     unixPerms: 770, filePerms: 775 : 770&775 = 770 (perms on directory is a superset)
	//     unixPerms: 770, filePerms: 770 : 770&770 = 770 (perms on directory is a superset)
	//     unixPerms: 770, filePerms: 750 : 770&750 = 750 (perms on directory is NOT a superset)
	// We also need to check if setgid bits are set in permissions of the directory.
	if (unixPerms&filePerm != unixPerms) || (fsInfo.Mode()&os.ModeSetgid == 0) {
		klog.V(4).InfoS("Performing recursive ownership change on rootDir because of mismatching mode", "path", rootDir)
		return true
	}
	return false
}


// readDirNames reads the directory named by dirname and returns
// a list of directory entries.
// We are not using filepath.readDirNames because we do not want to sort files found in a directory before changing
// permissions for performance reasons.
func readDirNames(dirname string) ([]string, error) {
	f, err := os.Open(dirname)
	if err != nil {
		return nil, err
	}
	names, err := f.Readdirnames(-1)
	f.Close()
	if err != nil {
		return nil, err
	}
	return names, nil
}


// walkDeep can be used to traverse directories and has two minor differences
// from filepath.Walk:
//   - List of files/dirs is not sorted for performance reasons
//   - callback walkFunc is invoked on root directory after visiting children dirs and files
func walkDeep(root string, walkFunc filepath.WalkFunc) error {
	info, err := os.Lstat(root)
	if err != nil {
		return walkFunc(root, nil, err)
	}
	return walk(root, info, walkFunc)
}


func walk(path string, info os.FileInfo, walkFunc filepath.WalkFunc) error {
	if !info.IsDir() {
		return walkFunc(path, info, nil)
	}
	names, err := readDirNames(path)
	if err != nil {
		return err
	}
	for _, name := range names {
		filename := filepath.Join(path, name)
		fileInfo, err := os.Lstat(filename)
		if err != nil {
			if err := walkFunc(filename, fileInfo, err); err != nil {
				return err
			}
		} else {
			err = walk(filename, fileInfo, walkFunc)
			if err != nil {
				return err
			}
		}
	}
	return walkFunc(path, info, nil)
}


func SetVolumeOwnership(mounter Mounter, dir string, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy, completeFunc func(types.CompleteFuncParam)) error {
	return nil
}


func (plugin *vsphereVolumePlugin) NewAttacher() (volume.Attacher, error) {
	vsphereCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
	if err != nil {
		return nil, err
	}

	return &vsphereVMDKAttacher{
		host:           plugin.host,
		vsphereVolumes: vsphereCloud,
	}, nil
}


func (plugin *vsphereVolumePlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
	return plugin.NewAttacher()
}


// Attaches the volume specified by the given spec to the given host.
// On success, returns the device path where the device was attached on the
// node.
// Callers are responsible for retryinging on failure.
// Callers are responsible for thread safety between concurrent attach and
// detach operations.
func (attacher *vsphereVMDKAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	klog.V(4).Infof("vSphere: Attach disk called for node %s", nodeName)

	// Keeps concurrent attach operations to same host atomic
	attachdetachMutex.LockKey(string(nodeName))
	defer attachdetachMutex.UnlockKey(string(nodeName))

	// vsphereCloud.AttachDisk checks if disk is already attached to host and
	// succeeds in that case, so no need to do that separately.
	diskUUID, err := attacher.vsphereVolumes.AttachDisk(volumeSource.VolumePath, volumeSource.StoragePolicyName, nodeName)
	if err != nil {
		klog.Errorf("Error attaching volume %q to node %q: %+v", volumeSource.VolumePath, nodeName, err)
		return "", err
	}

	return filepath.Join(diskByIDPath, diskSCSIPrefix+diskUUID), nil
}


func (attacher *vsphereVMDKAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.NodeName) (map[*volume.Spec]bool, error) {
	klog.Warningf("Attacher.VolumesAreAttached called for node %q - Please use BulkVerifyVolumes for vSphere", nodeName)
	volumeNodeMap := map[types.NodeName][]*volume.Spec{
		nodeName: specs,
	}
	nodeVolumesResult := make(map[*volume.Spec]bool)
	nodesVerificationMap, err := attacher.BulkVerifyVolumes(volumeNodeMap)
	if err != nil {
		klog.Errorf("Attacher.VolumesAreAttached - error checking volumes for node %q with %v", nodeName, err)
		return nodeVolumesResult, err
	}
	if result, ok := nodesVerificationMap[nodeName]; ok {
		return result, nil
	}
	return nodeVolumesResult, nil
}


func (attacher *vsphereVMDKAttacher) BulkVerifyVolumes(volumesByNode map[types.NodeName][]*volume.Spec) (map[types.NodeName]map[*volume.Spec]bool, error) {
	volumesAttachedCheck := make(map[types.NodeName]map[*volume.Spec]bool)
	volumePathsByNode := make(map[types.NodeName][]string)
	volumeSpecMap := make(map[string]*volume.Spec)

	for nodeName, volumeSpecs := range volumesByNode {
		for _, volumeSpec := range volumeSpecs {
			volumeSource, _, err := getVolumeSource(volumeSpec)
			if err != nil {
				klog.Errorf("Error getting volume (%q) source : %v", volumeSpec.Name(), err)
				continue
			}
			volPath := volumeSource.VolumePath
			volumePathsByNode[nodeName] = append(volumePathsByNode[nodeName], volPath)
			nodeVolume, nodeVolumeExists := volumesAttachedCheck[nodeName]
			if !nodeVolumeExists {
				nodeVolume = make(map[*volume.Spec]bool)
			}
			nodeVolume[volumeSpec] = true
			volumeSpecMap[volPath] = volumeSpec
			volumesAttachedCheck[nodeName] = nodeVolume
		}
	}
	attachedResult, err := attacher.vsphereVolumes.DisksAreAttached(volumePathsByNode)
	if err != nil {
		klog.Errorf("Error checking if volumes are attached to nodes: %+v. err: %v", volumePathsByNode, err)
		return volumesAttachedCheck, err
	}

	for nodeName, nodeVolumes := range attachedResult {
		for volumePath, attached := range nodeVolumes {
			if !attached {
				spec := volumeSpecMap[volumePath]
				setNodeVolume(volumesAttachedCheck, spec, nodeName, false)
			}
		}
	}
	return volumesAttachedCheck, nil
}


func (attacher *vsphereVMDKAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	if devicePath == "" {
		return "", fmt.Errorf("WaitForAttach failed for VMDK %q: devicePath is empty", volumeSource.VolumePath)
	}

	ticker := time.NewTicker(checkSleepDuration)
	defer ticker.Stop()

	timer := time.NewTimer(timeout)
	defer timer.Stop()

	for {
		select {
		case <-ticker.C:
			klog.V(5).Infof("Checking VMDK %q is attached", volumeSource.VolumePath)
			path, err := verifyDevicePath(devicePath)
			if err != nil {
				// Log error, if any, and continue checking periodically. See issue #11321
				klog.Warningf("Error verifying VMDK (%q) is attached: %v", volumeSource.VolumePath, err)
			} else if path != "" {
				// A device path has successfully been created for the VMDK
				klog.Infof("Successfully found attached VMDK %q.", volumeSource.VolumePath)
				return path, nil
			}
		case <-timer.C:
			return "", fmt.Errorf("could not find attached VMDK %q. Timeout waiting for mount paths to be created", volumeSource.VolumePath)
		}
	}
}


// GetDeviceMountPath returns a path where the device should
// point which should be bind mounted for individual volumes.
func (attacher *vsphereVMDKAttacher) GetDeviceMountPath(spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return makeGlobalPDPath(attacher.host, volumeSource.VolumePath), nil
}


// GetMountDeviceRefs finds all other references to the device referenced
// by deviceMountPath; returns a list of paths.
func (plugin *vsphereVolumePlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
	mounter := plugin.host.GetMounter(plugin.GetPluginName())
	return mounter.GetMountRefs(deviceMountPath)
}


// MountDevice mounts device to global mount point.
func (attacher *vsphereVMDKAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string, _ volume.DeviceMounterArgs) error {
	klog.Infof("vsphere MountDevice mount %s to %s", devicePath, deviceMountPath)
	mounter := attacher.host.GetMounter(vsphereVolumePluginName)
	notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath)
	if err != nil {
		if os.IsNotExist(err) {
			dir := deviceMountPath
			if runtime.GOOS == "windows" {
				dir = filepath.Dir(deviceMountPath)
			}
			if err := os.MkdirAll(dir, 0750); err != nil {
				klog.Errorf("Failed to create directory at %#v. err: %s", dir, err)
				return err
			}
			notMnt = true
		} else {
			return err
		}
	}

	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return err
	}

	options := []string{}

	if notMnt {
		diskMounter := volumeutil.NewSafeFormatAndMountFromHost(vsphereVolumePluginName, attacher.host)
		mountOptions := volumeutil.MountOptionFromSpec(spec, options...)
		err = diskMounter.FormatAndMount(devicePath, deviceMountPath, volumeSource.FSType, mountOptions)
		if err != nil {
			os.Remove(deviceMountPath)
			return err
		}
		klog.V(4).Infof("formatting spec %v devicePath %v deviceMountPath %v fs %v with options %+v", spec.Name(), devicePath, deviceMountPath, volumeSource.FSType, options)
	}
	return nil
}


func (plugin *vsphereVolumePlugin) NewDetacher() (volume.Detacher, error) {
	vsphereCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
	if err != nil {
		return nil, err
	}

	return &vsphereVMDKDetacher{
		mounter:        plugin.host.GetMounter(plugin.GetPluginName()),
		vsphereVolumes: vsphereCloud,
	}, nil
}


func (plugin *vsphereVolumePlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
	return plugin.NewDetacher()
}


// Detach the given device from the given node.
func (detacher *vsphereVMDKDetacher) Detach(volumeName string, nodeName types.NodeName) error {
	volPath := getVolPathfromVolumeName(volumeName)
	attached, newVolumePath, err := detacher.vsphereVolumes.DiskIsAttached(volPath, nodeName)
	if err != nil {
		// Log error and continue with detach
		klog.Errorf(
			"Error checking if volume (%q) is already attached to current node (%q). Will continue and try detach anyway. err=%v",
			volPath, nodeName, err)
	}

	if err == nil && !attached {
		// Volume is already detached from node.
		klog.Infof("detach operation was successful. volume %q is already detached from node %q.", volPath, nodeName)
		return nil
	}

	attachdetachMutex.LockKey(string(nodeName))
	defer attachdetachMutex.UnlockKey(string(nodeName))
	if err := detacher.vsphereVolumes.DetachDisk(newVolumePath, nodeName); err != nil {
		klog.Errorf("Error detaching volume %q: %v", volPath, err)
		return err
	}
	return nil
}


func (detacher *vsphereVMDKDetacher) UnmountDevice(deviceMountPath string) error {
	return mount.CleanupMountPoint(deviceMountPath, detacher.mounter, false)
}


func (plugin *vsphereVolumePlugin) CanAttach(spec *volume.Spec) (bool, error) {
	return true, nil
}


func (plugin *vsphereVolumePlugin) CanDeviceMount(spec *volume.Spec) (bool, error) {
	return true, nil
}


func setNodeVolume(

func setNodeVolume(
	nodeVolumeMap map[types.NodeName]map[*volume.Spec]bool,
	volumeSpec *volume.Spec,
	nodeName types.NodeName,
	check bool) {

	volumeMap := nodeVolumeMap[nodeName]
	if volumeMap == nil {
		volumeMap = make(map[*volume.Spec]bool)
		nodeVolumeMap[nodeName] = volumeMap
	}
	volumeMap[volumeSpec] = check
}


// This is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&vsphereVolumePlugin{}}
}


func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
	return host.GetPodVolumeDir(uid, utilstrings.EscapeQualifiedName(vsphereVolumePluginName), volName)
}


// vSphere Volume Plugin
func (plugin *vsphereVolumePlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	return nil
}


func (plugin *vsphereVolumePlugin) GetPluginName() string {
	return vsphereVolumePluginName
}


func (plugin *vsphereVolumePlugin) IsMigratedToCSI() bool {
	return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationvSphere)
}


func (plugin *vsphereVolumePlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return volumeSource.VolumePath, nil
}


func (plugin *vsphereVolumePlugin) CanSupport(spec *volume.Spec) bool {
	return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.VsphereVolume != nil) ||
		(spec.Volume != nil && spec.Volume.VsphereVolume != nil)
}


func (plugin *vsphereVolumePlugin) RequiresRemount(spec *volume.Spec) bool {
	return false
}


func (plugin *vsphereVolumePlugin) SupportsMountOption() bool {
	return true
}


func (plugin *vsphereVolumePlugin) SupportsBulkVolumeVerification() bool {
	return true
}


func (plugin *vsphereVolumePlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return false, nil
}


func (plugin *vsphereVolumePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
	return plugin.newMounterInternal(spec, pod.UID, &VsphereDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName()))
}


func (plugin *vsphereVolumePlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	return plugin.newUnmounterInternal(volName, podUID, &VsphereDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName()))
}


func (plugin *vsphereVolumePlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager vdManager, mounter mount.Interface) (volume.Mounter, error) {
	vvol, _, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}

	volPath := vvol.VolumePath
	fsType := vvol.FSType

	return &vsphereVolumeMounter{
		vsphereVolume: &vsphereVolume{
			podUID:          podUID,
			volName:         spec.Name(),
			volPath:         volPath,
			manager:         manager,
			mounter:         mounter,
			plugin:          plugin,
			MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, spec.Name(), plugin.host)),
		},
		fsType:       fsType,
		diskMounter:  util.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host),
		mountOptions: util.MountOptionFromSpec(spec),
	}, nil
}


func (plugin *vsphereVolumePlugin) newUnmounterInternal(volName string, podUID types.UID, manager vdManager, mounter mount.Interface) (volume.Unmounter, error) {
	return &vsphereVolumeUnmounter{
		&vsphereVolume{
			podUID:          podUID,
			volName:         volName,
			manager:         manager,
			mounter:         mounter,
			plugin:          plugin,
			MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)),
		}}, nil
}


func (plugin *vsphereVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
	mounter := plugin.host.GetMounter(plugin.GetPluginName())
	kvh, ok := plugin.host.(volume.KubeletVolumeHost)
	if !ok {
		return volume.ReconstructedVolume{}, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
	}
	hu := kvh.GetHostUtil()
	pluginMntDir := util.GetPluginMountDir(plugin.host, plugin.GetPluginName())
	volumePath, err := hu.GetDeviceNameFromMount(mounter, mountPath, pluginMntDir)
	if err != nil {
		return volume.ReconstructedVolume{}, err
	}
	volumePath = strings.Replace(volumePath, "\\040", " ", -1)
	klog.V(5).Infof("vSphere volume path is %q", volumePath)
	vsphereVolume := &v1.Volume{
		Name: volumeName,
		VolumeSource: v1.VolumeSource{
			VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
				VolumePath: volumePath,
			},
		},
	}
	return volume.ReconstructedVolume{
		Spec: volume.NewSpecFromVolume(vsphereVolume),
	}, nil
}


func (b *vsphereVolumeMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		SELinuxRelabel: true,
		Managed:        true,
	}
}


// SetUp attaches the disk and bind mounts to the volume path.
func (b *vsphereVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error {
	return b.SetUpAt(b.GetPath(), mounterArgs)
}


// SetUp attaches the disk and bind mounts to the volume path.
func (b *vsphereVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	klog.V(5).Infof("vSphere volume setup %s to %s", b.volPath, dir)

	// TODO: handle failed mounts here.
	notmnt, err := b.mounter.IsLikelyNotMountPoint(dir)
	if err != nil && !os.IsNotExist(err) {
		klog.V(4).Infof("IsLikelyNotMountPoint failed: %v", err)
		return err
	}
	if !notmnt {
		klog.V(4).Infof("Something is already mounted to target %s", dir)
		return nil
	}

	if runtime.GOOS != "windows" {
		// On Windows, Mount will create the parent of dir and mklink (create a symbolic link) at dir later, so don't create a
		// directory at dir now. Otherwise mklink will error: "Cannot create a file when that file already exists".
		if err := os.MkdirAll(dir, 0750); err != nil {
			klog.Errorf("Could not create directory %s: %v", dir, err)
			return err
		}
	}

	options := []string{"bind"}

	// Perform a bind mount to the full path to allow duplicate mounts of the same PD.
	globalPDPath := makeGlobalPDPath(b.plugin.host, b.volPath)
	mountOptions := util.JoinMountOptions(options, b.mountOptions)
	err = b.mounter.MountSensitiveWithoutSystemd(globalPDPath, dir, "", mountOptions, nil)
	if err != nil {
		notmnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir)
		if mntErr != nil {
			klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
			return err
		}
		if !notmnt {
			if mntErr = b.mounter.Unmount(dir); mntErr != nil {
				klog.Errorf("Failed to unmount: %v", mntErr)
				return err
			}
			notmnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir)
			if mntErr != nil {
				klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
				return err
			}
			if !notmnt {
				klog.Errorf("%s is still mounted, despite call to unmount().  Will try again next sync loop.", b.GetPath())
				return err
			}
		}
		os.Remove(dir)
		return err
	}
	volume.SetVolumeOwnership(b, dir, mounterArgs.FsGroup, mounterArgs.FSGroupChangePolicy, util.FSGroupCompleteHook(b.plugin, nil))
	klog.V(3).Infof("vSphere volume %s mounted to %s", b.volPath, dir)

	return nil
}


// Unmounts the bind mount, and detaches the disk only if the PD
// resource was the last reference to that disk on the kubelet.
func (v *vsphereVolumeUnmounter) TearDown() error {
	return v.TearDownAt(v.GetPath())
}


// Unmounts the bind mount, and detaches the disk only if the PD
// resource was the last reference to that disk on the kubelet.
func (v *vsphereVolumeUnmounter) TearDownAt(dir string) error {
	return mount.CleanupMountPoint(dir, v.mounter, false)
}


func makeGlobalPDPath(host volume.VolumeHost, devName string) string {
	return filepath.Join(host.GetPluginDir(vsphereVolumePluginName), util.MountsInGlobalPDPath, devName)
}


func (vv *vsphereVolume) GetPath() string {
	name := vsphereVolumePluginName
	return vv.plugin.host.GetPodVolumeDir(vv.podUID, utilstrings.EscapeQualifiedName(name), vv.volName)
}


// vSphere Persistent Volume Plugin
func (plugin *vsphereVolumePlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
	return []v1.PersistentVolumeAccessMode{
		v1.ReadWriteOnce,
	}
}


func (plugin *vsphereVolumePlugin) NewDeleter(logger klog.Logger, spec *volume.Spec) (volume.Deleter, error) {
	return plugin.newDeleterInternal(spec, &VsphereDiskUtil{})
}


func (plugin *vsphereVolumePlugin) newDeleterInternal(spec *volume.Spec, manager vdManager) (volume.Deleter, error) {
	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.VsphereVolume == nil {
		return nil, fmt.Errorf("spec.PersistentVolumeSource.VsphereVolume is nil")
	}
	return &vsphereVolumeDeleter{
		&vsphereVolume{
			volName: spec.Name(),
			volPath: spec.PersistentVolume.Spec.VsphereVolume.VolumePath,
			manager: manager,
			plugin:  plugin,
		}}, nil
}


func (r *vsphereVolumeDeleter) Delete() error {
	return r.manager.DeleteVolume(r)
}


func (plugin *vsphereVolumePlugin) NewProvisioner(logger klog.Logger, options volume.VolumeOptions) (volume.Provisioner, error) {
	return plugin.newProvisionerInternal(options, &VsphereDiskUtil{})
}


func (plugin *vsphereVolumePlugin) newProvisionerInternal(options volume.VolumeOptions, manager vdManager) (volume.Provisioner, error) {
	return &vsphereVolumeProvisioner{
		vsphereVolume: &vsphereVolume{
			manager: manager,
			plugin:  plugin,
		},
		options: options,
	}, nil
}


func (v *vsphereVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTopologies []v1.TopologySelectorTerm) (*v1.PersistentVolume, error) {
	if !util.ContainsAllAccessModes(v.plugin.GetAccessModes(), v.options.PVC.Spec.AccessModes) {
		return nil, fmt.Errorf("invalid AccessModes %v: only AccessModes %v are supported", v.options.PVC.Spec.AccessModes, v.plugin.GetAccessModes())
	}
	klog.V(1).Infof("Provision with selectedNode: %s and allowedTopologies : %s", getNodeName(selectedNode), allowedTopologies)
	selectedZones, err := volumehelpers.ZonesFromAllowedTopologies(allowedTopologies)
	if err != nil {
		return nil, err
	}

	klog.V(4).Infof("Selected zones for volume : %s", selectedZones)
	volSpec, err := v.manager.CreateVolume(v, selectedNode, selectedZones.List())
	if err != nil {
		return nil, err
	}

	if volSpec.Fstype == "" {
		volSpec.Fstype = "ext4"
	}

	volumeMode := v.options.PVC.Spec.VolumeMode
	if volumeMode != nil && *volumeMode == v1.PersistentVolumeBlock {
		klog.V(5).Infof("vSphere block volume should not have any FSType")
		volSpec.Fstype = ""
	}

	pv := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name:   v.options.PVName,
			Labels: map[string]string{},
			Annotations: map[string]string{
				util.VolumeDynamicallyCreatedByKey: "vsphere-volume-dynamic-provisioner",
			},
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeReclaimPolicy: v.options.PersistentVolumeReclaimPolicy,
			AccessModes:                   v.options.PVC.Spec.AccessModes,
			Capacity: v1.ResourceList{
				v1.ResourceName(v1.ResourceStorage): resource.MustParse(fmt.Sprintf("%dKi", volSpec.Size)),
			},
			VolumeMode: volumeMode,
			PersistentVolumeSource: v1.PersistentVolumeSource{
				VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
					VolumePath:        volSpec.Path,
					FSType:            volSpec.Fstype,
					StoragePolicyName: volSpec.StoragePolicyName,
					StoragePolicyID:   volSpec.StoragePolicyID,
				},
			},
			MountOptions: v.options.MountOptions,
		},
	}
	if len(v.options.PVC.Spec.AccessModes) == 0 {
		pv.Spec.AccessModes = v.plugin.GetAccessModes()
	}

	labels := volSpec.Labels
	requirements := make([]v1.NodeSelectorRequirement, 0)
	if len(labels) != 0 {
		if pv.Labels == nil {
			pv.Labels = make(map[string]string)
		}
		for k, v := range labels {
			pv.Labels[k] = v
			var values []string
			if k == v1.LabelTopologyZone || k == v1.LabelFailureDomainBetaZone {
				values, err = volumehelpers.LabelZonesToList(v)
				if err != nil {
					return nil, fmt.Errorf("failed to convert label string for Zone: %s to a List: %v", v, err)
				}
			} else {
				values = []string{v}
			}
			requirements = append(requirements, v1.NodeSelectorRequirement{Key: k, Operator: v1.NodeSelectorOpIn, Values: values})
		}
	}

	if len(requirements) > 0 {
		pv.Spec.NodeAffinity = new(v1.VolumeNodeAffinity)
		pv.Spec.NodeAffinity.Required = new(v1.NodeSelector)
		pv.Spec.NodeAffinity.Required.NodeSelectorTerms = make([]v1.NodeSelectorTerm, 1)
		pv.Spec.NodeAffinity.Required.NodeSelectorTerms[0].MatchExpressions = requirements
	}

	return pv, nil
}


func getVolumeSource(

func getVolumeSource(
	spec *volume.Spec) (*v1.VsphereVirtualDiskVolumeSource, bool, error) {
	if spec.Volume != nil && spec.Volume.VsphereVolume != nil {
		return spec.Volume.VsphereVolume, spec.ReadOnly, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.VsphereVolume != nil {
		return spec.PersistentVolume.Spec.VsphereVolume, spec.ReadOnly, nil
	}

	return nil, false, fmt.Errorf("spec does not reference a VSphere volume type")
}


func getNodeName(node *v1.Node) string

func getNodeName(node *v1.Node) string {
	if node == nil {
		return ""
	}
	return node.Name
}


func (plugin *vsphereVolumePlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mapPath string) (*volume.Spec, error) {

	pluginDir := plugin.host.GetPluginDir(plugin.GetPluginName())
	blkUtil := volumepathhandler.NewBlockVolumePathHandler()
	globalMapPathUUID, err := blkUtil.FindGlobalMapPathUUIDFromPod(pluginDir, mapPath, podUID)
	if err != nil {
		klog.Errorf("Failed to find GlobalMapPathUUID from Pod: %s with error: %+v", podUID, err)
		return nil, err
	}
	klog.V(5).Infof("globalMapPathUUID: %v", globalMapPathUUID)
	globalMapPath := filepath.Dir(globalMapPathUUID)
	if len(globalMapPath) <= 1 {
		return nil, fmt.Errorf("failed to get volume plugin information from globalMapPathUUID: %v", globalMapPathUUID)
	}
	return getVolumeSpecFromGlobalMapPath(volumeName, globalMapPath)
}


func getVolumeSpecFromGlobalMapPath(volumeName, globalMapPath string) (*volume.Spec, error) {
	// Construct volume spec from globalMapPath
	// globalMapPath example:
	//   plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumeID}
	//   plugins/kubernetes.io/vsphere-volume/volumeDevices/[datastore1]\\040volumes/myDisk
	volPath := filepath.Base(globalMapPath)
	volPath = strings.Replace(volPath, "\\040", "", -1)
	if len(volPath) <= 1 {
		return nil, fmt.Errorf("failed to get volume path from global path=%s", globalMapPath)
	}
	block := v1.PersistentVolumeBlock
	vsphereVolume := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: volumeName,
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeSource: v1.PersistentVolumeSource{
				VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
					VolumePath: volPath,
				},
			},
			VolumeMode: &block,
		},
	}
	return volume.NewSpecFromPersistentVolume(vsphereVolume, true), nil
}


func (plugin *vsphereVolumePlugin) NewBlockVolumeMapper(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.BlockVolumeMapper, error) {
	// If this called via GenerateUnmapDeviceFunc(), pod is nil.
	// Pass empty string as dummy uid since uid isn't used in the case.
	var uid types.UID
	if pod != nil {
		uid = pod.UID
	}
	return plugin.newBlockVolumeMapperInternal(spec, uid, &VsphereDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName()))
}


func (plugin *vsphereVolumePlugin) newBlockVolumeMapperInternal(spec *volume.Spec, podUID types.UID, manager vdManager, mounter mount.Interface) (volume.BlockVolumeMapper, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		klog.Errorf("Failed to get Volume source from volume Spec: %+v with error: %+v", *spec, err)
		return nil, err
	}
	volPath := volumeSource.VolumePath
	mapper := &vsphereBlockVolumeMapper{
		vsphereVolume: &vsphereVolume{
			volName:         spec.Name(),
			podUID:          podUID,
			volPath:         volPath,
			manager:         manager,
			mounter:         mounter,
			plugin:          plugin,
			MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, spec.Name(), plugin.host)),
		},
	}

	blockPath, err := mapper.GetGlobalMapPath(spec)
	if err != nil {
		return nil, fmt.Errorf("failed to get device path: %v", err)
	}
	mapper.MetricsProvider = volume.NewMetricsBlock(filepath.Join(blockPath, string(podUID)))

	return mapper, nil
}


func (plugin *vsphereVolumePlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (volume.BlockVolumeUnmapper, error) {
	return plugin.newUnmapperInternal(volName, podUID, &VsphereDiskUtil{})
}


func (plugin *vsphereVolumePlugin) newUnmapperInternal(volName string, podUID types.UID, manager vdManager) (volume.BlockVolumeUnmapper, error) {
	return &vsphereBlockVolumeUnmapper{
		vsphereVolume: &vsphereVolume{
			volName: volName,
			podUID:  podUID,
			volPath: volName,
			manager: manager,
			plugin:  plugin,
		},
	}, nil
}


// GetGlobalMapPath returns global map path and error
// path: plugins/kubernetes.io/{PluginName}/volumeDevices/volumePath
func (v *vsphereVolume) GetGlobalMapPath(spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}
	return filepath.Join(v.plugin.host.GetVolumeDevicePluginDir(vsphereVolumePluginName), string(volumeSource.VolumePath)), nil
}


func (v *vsphereVolume) GetPodDeviceMapPath() (string, string) {
	return v.plugin.host.GetPodVolumeDeviceDir(v.podUID, utilstrings.EscapeQualifiedName(vsphereVolumePluginName)), v.volName
}


// SupportsMetrics returns true for vsphereBlockVolumeMapper as it initializes the
// MetricsProvider.
func (vbvm *vsphereBlockVolumeMapper) SupportsMetrics() bool {
	return true
}


// CreateVolume creates a vSphere volume.
func (util *VsphereDiskUtil) CreateVolume(v *vsphereVolumeProvisioner, selectedNode *v1.Node, selectedZone []string) (volSpec *VolumeSpec, err error) {
	var fstype string
	cloud, err := getCloudProvider(v.plugin.host.GetCloudProvider())
	if err != nil {
		return nil, err
	}

	capacity := v.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	// vSphere works with KiB, but its minimum allocation unit is 1 MiB
	volSizeMiB, err := volumehelpers.RoundUpToMiBInt(capacity)
	if err != nil {
		return nil, err
	}
	volSizeKiB := volSizeMiB * 1024
	name := volumeutil.GenerateVolumeName(v.options.ClusterName, v.options.PVName, maxVolumeLength)
	volumeOptions := &vclib.VolumeOptions{
		CapacityKB: volSizeKiB,
		Tags:       *v.options.CloudTags,
		Name:       name,
	}

	volumeOptions.Zone = selectedZone
	volumeOptions.SelectedNode = selectedNode
	// Apply Parameters (case-insensitive). We leave validation of
	// the values to the cloud provider.
	for parameter, value := range v.options.Parameters {
		switch strings.ToLower(parameter) {
		case diskformat:
			volumeOptions.DiskFormat = value
		case datastore:
			volumeOptions.Datastore = value
		case volume.VolumeParameterFSType:
			fstype = value
			klog.V(4).Infof("Setting fstype as %q", fstype)
		case StoragePolicyName:
			volumeOptions.StoragePolicyName = value
			klog.V(4).Infof("Setting StoragePolicyName as %q", volumeOptions.StoragePolicyName)
		case HostFailuresToTolerateCapability, ForceProvisioningCapability,
			CacheReservationCapability, DiskStripesCapability,
			ObjectSpaceReservationCapability, IopsLimitCapability:
			capabilityData, err := validateVSANCapability(strings.ToLower(parameter), value)
			if err != nil {
				return nil, err
			}
			volumeOptions.VSANStorageProfileData += capabilityData
		default:
			return nil, fmt.Errorf("invalid option %q for volume plugin %s", parameter, v.plugin.GetPluginName())
		}
	}

	if volumeOptions.VSANStorageProfileData != "" {
		if volumeOptions.StoragePolicyName != "" {
			return nil, fmt.Errorf("cannot specify storage policy capabilities along with storage policy name. Please specify only one")
		}
		volumeOptions.VSANStorageProfileData = "(" + volumeOptions.VSANStorageProfileData + ")"
	}
	klog.V(4).Infof("VSANStorageProfileData in vsphere volume %q", volumeOptions.VSANStorageProfileData)
	// TODO: implement PVC.Selector parsing
	if v.options.PVC.Spec.Selector != nil {
		return nil, fmt.Errorf("claim.Spec.Selector is not supported for dynamic provisioning on vSphere")
	}

	vmDiskPath, err := cloud.CreateVolume(volumeOptions)
	if err != nil {
		return nil, err
	}
	labels, err := cloud.GetVolumeLabels(vmDiskPath)
	if err != nil {
		return nil, err
	}
	volSpec = &VolumeSpec{
		Path:              vmDiskPath,
		Size:              volSizeKiB,
		Fstype:            fstype,
		StoragePolicyName: volumeOptions.StoragePolicyName,
		StoragePolicyID:   volumeOptions.StoragePolicyID,
		Labels:            labels,
	}
	klog.V(2).Infof("Successfully created vsphere volume %s", name)
	return volSpec, nil
}


// DeleteVolume deletes a vSphere volume.
func (util *VsphereDiskUtil) DeleteVolume(vd *vsphereVolumeDeleter) error {
	cloud, err := getCloudProvider(vd.plugin.host.GetCloudProvider())
	if err != nil {
		return err
	}

	if err = cloud.DeleteVolume(vd.volPath); err != nil {
		klog.V(2).Infof("Error deleting vsphere volume %s: %v", vd.volPath, err)
		return err
	}
	klog.V(2).Infof("Successfully deleted vsphere volume %s", vd.volPath)
	return nil
}


func getVolPathfromVolumeName(deviceMountPath string) string {
	// Assumption: No file or folder is named starting with '[' in datastore
	volPath := deviceMountPath[strings.LastIndex(deviceMountPath, "["):]
	// space between datastore and vmdk name in volumePath is encoded as '\040' when returned by GetMountRefs().
	// volumePath eg: "[local] xxx.vmdk" provided to attach/mount
	// replacing \040 with space to match the actual volumePath
	return strings.Replace(volPath, "\\040", " ", -1)
}


func getCloudProvider(cloud cloudprovider.Interface) (*vsphere.VSphere, error) {
	if cloud == nil {
		klog.Errorf("Cloud provider not initialized properly")
		return nil, errors.New("cloud provider not initialized properly")
	}

	vs, ok := cloud.(*vsphere.VSphere)
	if !ok || vs == nil {
		return nil, errors.New("invalid cloud provider: expected vSphere")
	}
	return vs, nil
}


// Validate the capability requirement for the user specified policy attributes.
func validateVSANCapability(capabilityName string, capabilityValue string) (string, error) {
	var capabilityData string
	capabilityIntVal, ok := verifyCapabilityValueIsInteger(capabilityValue)
	if !ok {
		return "", fmt.Errorf("invalid value for %s. The capabilityValue: %s must be a valid integer value", capabilityName, capabilityValue)
	}
	switch strings.ToLower(capabilityName) {
	case HostFailuresToTolerateCapability:
		if capabilityIntVal >= HostFailuresToTolerateCapabilityMin && capabilityIntVal <= HostFailuresToTolerateCapabilityMax {
			capabilityData = " (\"hostFailuresToTolerate\" i" + capabilityValue + ")"
		} else {
			return "", fmt.Errorf(`invalid value for hostFailuresToTolerate.
				The default value is %d, minimum value is %d and maximum value is %d`,
				1, HostFailuresToTolerateCapabilityMin, HostFailuresToTolerateCapabilityMax)
		}
	case ForceProvisioningCapability:
		if capabilityIntVal >= ForceProvisioningCapabilityMin && capabilityIntVal <= ForceProvisioningCapabilityMax {
			capabilityData = " (\"forceProvisioning\" i" + capabilityValue + ")"
		} else {
			return "", fmt.Errorf(`invalid value for forceProvisioning.
				The value can be either %d or %d`,
				ForceProvisioningCapabilityMin, ForceProvisioningCapabilityMax)
		}
	case CacheReservationCapability:
		if capabilityIntVal >= CacheReservationCapabilityMin && capabilityIntVal <= CacheReservationCapabilityMax {
			capabilityData = " (\"cacheReservation\" i" + strconv.Itoa(capabilityIntVal*10000) + ")"
		} else {
			return "", fmt.Errorf(`invalid value for cacheReservation.
				The minimum percentage is %d and maximum percentage is %d`,
				CacheReservationCapabilityMin, CacheReservationCapabilityMax)
		}
	case DiskStripesCapability:
		if capabilityIntVal >= DiskStripesCapabilityMin && capabilityIntVal <= DiskStripesCapabilityMax {
			capabilityData = " (\"stripeWidth\" i" + capabilityValue + ")"
		} else {
			return "", fmt.Errorf(`invalid value for diskStripes.
				The minimum value is %d and maximum value is %d`,
				DiskStripesCapabilityMin, DiskStripesCapabilityMax)
		}
	case ObjectSpaceReservationCapability:
		if capabilityIntVal >= ObjectSpaceReservationCapabilityMin && capabilityIntVal <= ObjectSpaceReservationCapabilityMax {
			capabilityData = " (\"proportionalCapacity\" i" + capabilityValue + ")"
		} else {
			return "", fmt.Errorf(`invalid value for ObjectSpaceReservation.
				The minimum percentage is %d and maximum percentage is %d`,
				ObjectSpaceReservationCapabilityMin, ObjectSpaceReservationCapabilityMax)
		}
	case IopsLimitCapability:
		if capabilityIntVal >= IopsLimitCapabilityMin {
			capabilityData = " (\"iopsLimit\" i" + capabilityValue + ")"
		} else {
			return "", fmt.Errorf(`invalid value for iopsLimit.
				The value should be greater than %d`, IopsLimitCapabilityMin)
		}
	}
	return capabilityData, nil
}


// Verify if the capability value is of type integer.
func verifyCapabilityValueIsInteger(capabilityValue string) (int, bool) {
	i, err := strconv.Atoi(capabilityValue)
	if err != nil {
		return -1, false
	}
	return i, true
}


func verifyDevicePath(path string) (string, error) {
	if pathExists, err := mount.PathExists(path); err != nil {
		return "", fmt.Errorf("error checking if path exists: %w", err)
	} else if pathExists {
		return path, nil
	}

	return "", nil
}


func verifyDevicePath(path string) (string, error) {
	return "", errors.New("unsupported")
}


func verifyDevicePath(path string) (string, error) {
	if !strings.Contains(path, diskByIDPath) {
		// If this volume has already been mounted then
		// its devicePath will have already been converted to a disk number
		klog.V(4).Infof("Found vSphere disk attached with disk number %v", path)
		return path, nil
	}
	cmd := exec.Command("powershell", "/c", "Get-Disk | Select Number, SerialNumber | ConvertTo-JSON")
	output, err := cmd.Output()
	if err != nil {
		klog.Errorf("Get-Disk failed, error: %v, output: %q", err, string(output))
		return "", err
	}

	var results []diskInfoResult
	if err = json.Unmarshal(output, &results); err != nil {
		klog.Errorf("Failed to unmarshal Get-Disk json, output: %q", string(output))
		return "", err
	}
	serialNumber := strings.TrimPrefix(path, diskByIDPath+diskSCSIPrefix)
	for _, v := range results {
		if v.SerialNumber == serialNumber {
			klog.V(4).Infof("Found vSphere disk attached with serial %v", serialNumber)
			return v.Number.String(), nil
		}
	}

	return "", fmt.Errorf("unable to find vSphere disk with serial %v", serialNumber)
}


// InitService is the entry point for running the daemon as a Windows
// service. It returns an indication of whether it is running as a service;
// and an error.
func InitService(serviceName string) error {
	h := &handler{
		tosvc:   make(chan bool),
		fromsvc: make(chan error),
	}

	var err error
	go func() {
		err = svc.Run(serviceName, h)
		h.fromsvc <- err
	}()

	// Wait for the first signal from the service handler.
	err = <-h.fromsvc
	if err != nil {
		return err
	}
	klog.Infof("Running %s as a Windows service!", serviceName)
	return nil
}


func (h *handler) Execute(_ []string, r <-chan svc.ChangeRequest, s chan<- svc.Status) (bool, uint32) {
	s <- svc.Status{State: svc.StartPending, Accepts: 0}
	// Unblock initService()
	h.fromsvc <- nil

	s <- svc.Status{State: svc.Running, Accepts: svc.AcceptStop | svc.AcceptShutdown | svc.Accepted(windows.SERVICE_ACCEPT_PARAMCHANGE)}
	klog.Infof("Service running")
Loop:
	for {
		select {
		case <-h.tosvc:
			break Loop
		case c := <-r:
			switch c.Cmd {
			case svc.Cmd(windows.SERVICE_CONTROL_PARAMCHANGE):
				s <- c.CurrentStatus
			case svc.Interrogate:
				s <- c.CurrentStatus
			case svc.Stop, svc.Shutdown:
				klog.Infof("Service stopping")
				// We need to translate this request into a signal that can be handled by the signal handler
				// handling shutdowns normally (currently apiserver/pkg/server/signal.go).
				// If we do not do this, our main threads won't be notified of the upcoming shutdown.
				// Since Windows services do not use any console, we cannot simply generate a CTRL_BREAK_EVENT
				// but need a dedicated notification mechanism.
				graceful := server.RequestShutdown()

				// Free up the control handler and let us terminate as gracefully as possible.
				// If that takes too long, the service controller will kill the remaining threads.
				// As per https://docs.microsoft.com/en-us/windows/desktop/services/service-control-handler-function
				s <- svc.Status{State: svc.StopPending}

				// If we cannot exit gracefully, we really only can exit our process, so at least the
				// service manager will think that we gracefully exited. At the time of writing this comment this is
				// needed for applications that do not use signals (e.g. kube-proxy)
				if !graceful {
					go func() {
						// Ensure the SCM was notified (The operation above (send to s) was received and communicated to the
						// service control manager - so it doesn't look like the service crashes)
						time.Sleep(1 * time.Second)
						os.Exit(0)
					}()
				}
				break Loop
			}
		}
	}

	return false, 0
}


// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		return NewAlwaysAdmit(), nil
	})
}


// Admit makes an admission decision based on the request attributes
func (alwaysAdmit) Admit(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) (err error) {
	return nil
}


// Validate makes an admission decision based on the request attributes.  It is NOT allowed to mutate.
func (alwaysAdmit) Validate(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) (err error) {
	return nil
}


// Handles returns true if this admission controller can handle the given operation
// where operation can be one of CREATE, UPDATE, DELETE, or CONNECT
func (alwaysAdmit) Handles(operation admission.Operation) bool {
	return true
}


// NewAlwaysAdmit creates a new always admit admission handler
func NewAlwaysAdmit() admission.Interface {
	// DEPRECATED: AlwaysAdmit admit all admission request, it is no use.
	klog.Warningf("%s admission controller is deprecated. "+
		"Please remove this controller from your configuration files and scripts.", PluginName)
	return new(alwaysAdmit)
}


// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		return NewAlwaysPullImages(), nil
	})
}


// Admit makes an admission decision based on the request attributes
func (a *AlwaysPullImages) Admit(ctx context.Context, attributes admission.Attributes, o admission.ObjectInterfaces) (err error) {
	// Ignore all calls to subresources or resources other than pods.
	if shouldIgnore(attributes) {
		return nil
	}
	pod, ok := attributes.GetObject().(*api.Pod)
	if !ok {
		return apierrors.NewBadRequest("Resource was marked with kind Pod but was unable to be converted")
	}

	pods.VisitContainersWithPath(&pod.Spec, field.NewPath("spec"), func(c *api.Container, _ *field.Path) bool {
		c.ImagePullPolicy = api.PullAlways
		return true
	})

	return nil
}


// Validate makes sure that all containers are set to always pull images
func (*AlwaysPullImages) Validate(ctx context.Context, attributes admission.Attributes, o admission.ObjectInterfaces) (err error) {
	if shouldIgnore(attributes) {
		return nil
	}

	pod, ok := attributes.GetObject().(*api.Pod)
	if !ok {
		return apierrors.NewBadRequest("Resource was marked with kind Pod but was unable to be converted")
	}

	var allErrs []error
	pods.VisitContainersWithPath(&pod.Spec, field.NewPath("spec"), func(c *api.Container, p *field.Path) bool {
		if c.ImagePullPolicy != api.PullAlways {
			allErrs = append(allErrs, admission.NewForbidden(attributes,
				field.NotSupported(p.Child("imagePullPolicy"), c.ImagePullPolicy, []string{string(api.PullAlways)}),
			))
		}
		return true
	})
	if len(allErrs) > 0 {
		return utilerrors.NewAggregate(allErrs)
	}

	return nil
}


// check if it's update and it doesn't change the images referenced by the pod spec
func isUpdateWithNoNewImages(attributes admission.Attributes) bool {
	if attributes.GetOperation() != admission.Update {
		return false
	}

	pod, ok := attributes.GetObject().(*api.Pod)
	if !ok {
		klog.Warningf("Resource was marked with kind Pod but pod was unable to be converted.")
		return false
	}

	oldPod, ok := attributes.GetOldObject().(*api.Pod)
	if !ok {
		klog.Warningf("Resource was marked with kind Pod but old pod was unable to be converted.")
		return false
	}

	oldImages := sets.NewString()
	pods.VisitContainersWithPath(&oldPod.Spec, field.NewPath("spec"), func(c *api.Container, _ *field.Path) bool {
		oldImages.Insert(c.Image)
		return true
	})

	hasNewImage := false
	pods.VisitContainersWithPath(&pod.Spec, field.NewPath("spec"), func(c *api.Container, _ *field.Path) bool {
		if !oldImages.Has(c.Image) {
			hasNewImage = true
		}
		return !hasNewImage
	})
	return !hasNewImage
}


func shouldIgnore(attributes admission.Attributes) bool {
	// Ignore all calls to subresources or resources other than pods.
	if len(attributes.GetSubresource()) != 0 || attributes.GetResource().GroupResource() != api.Resource("pods") {
		return true
	}

	if isUpdateWithNoNewImages(attributes) {
		return true
	}
	return false
}


// NewAlwaysPullImages creates a new always pull images admission control handler
func NewAlwaysPullImages() *AlwaysPullImages {
	return &AlwaysPullImages{
		Handler: admission.NewHandler(admission.Create, admission.Update),
	}
}


// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		return NewInterPodAntiAffinity(), nil
	})
}


func NewInterPodAntiAffinity() *Plugin

// NewInterPodAntiAffinity creates a new instance of the LimitPodHardAntiAffinityTopology admission controller
func NewInterPodAntiAffinity() *Plugin {
	return &Plugin{
		Handler: admission.NewHandler(admission.Create, admission.Update),
	}
}


// Validate will deny any pod that defines AntiAffinity topology key other than v1.LabelHostname i.e. "kubernetes.io/hostname"
// in  requiredDuringSchedulingRequiredDuringExecution and requiredDuringSchedulingIgnoredDuringExecution.
func (p *Plugin) Validate(ctx context.Context, attributes admission.Attributes, o admission.ObjectInterfaces) (err error) {
	// Ignore all calls to subresources or resources other than pods.
	if len(attributes.GetSubresource()) != 0 || attributes.GetResource().GroupResource() != api.Resource("pods") {
		return nil
	}
	pod, ok := attributes.GetObject().(*api.Pod)
	if !ok {
		return apierrors.NewBadRequest("Resource was marked with kind Pod but was unable to be converted")
	}
	affinity := pod.Spec.Affinity
	if affinity != nil && affinity.PodAntiAffinity != nil {
		var podAntiAffinityTerms []api.PodAffinityTerm
		if len(affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution) != 0 {
			podAntiAffinityTerms = affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution
		}
		// TODO: Uncomment this block when implement RequiredDuringSchedulingRequiredDuringExecution.
		//if len(affinity.PodAntiAffinity.RequiredDuringSchedulingRequiredDuringExecution) != 0 {
		//        podAntiAffinityTerms = append(podAntiAffinityTerms, affinity.PodAntiAffinity.RequiredDuringSchedulingRequiredDuringExecution...)
		//}
		for _, v := range podAntiAffinityTerms {
			if v.TopologyKey != v1.LabelHostname {
				return apierrors.NewForbidden(attributes.GetResource().GroupResource(), pod.Name, fmt.Errorf("affinity.PodAntiAffinity.RequiredDuringScheduling has TopologyKey %v but only key %v is allowed", v.TopologyKey, v1.LabelHostname))
			}
		}
	}
	return nil
}


// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		return NewPlugin(), nil
	})
}


// SetAuthorizer sets the authorizer.
func (p *Plugin) SetAuthorizer(authz authorizer.Authorizer) {
	p.authz = authz
}


// ValidateInitialization ensures an authorizer is set.
func (p *Plugin) ValidateInitialization() error {
	if p.authz == nil {
		return fmt.Errorf("%s requires an authorizer", PluginName)
	}
	return nil
}


func NewPlugin() *Plugin

// NewPlugin creates a new CSR approval admission plugin
func NewPlugin() *Plugin {
	return &Plugin{
		Handler: admission.NewHandler(admission.Update),
	}
}


// Validate verifies that the requesting user has permission to approve
// CertificateSigningRequests for the specified signerName.
func (p *Plugin) Validate(ctx context.Context, a admission.Attributes, _ admission.ObjectInterfaces) error {
	// Ignore all calls to anything other than 'certificatesigningrequests/approval'.
	// Ignore all operations other than UPDATE.
	if a.GetSubresource() != "approval" ||
		a.GetResource().GroupResource() != csrGroupResource {
		return nil
	}

	// We check permissions against the *old* version of the resource, in case
	// a user is attempting to update the SignerName when calling the approval
	// endpoint (which is an invalid/not allowed operation)
	csr, ok := a.GetOldObject().(*api.CertificateSigningRequest)
	if !ok {
		return admission.NewForbidden(a, fmt.Errorf("expected type CertificateSigningRequest, got: %T", a.GetOldObject()))
	}

	if !certificates.IsAuthorizedForSignerName(ctx, p.authz, a.GetUserInfo(), "approve", csr.Spec.SignerName) {
		klog.V(4).Infof("user not permitted to approve CertificateSigningRequest %q with signerName %q", csr.Name, csr.Spec.SignerName)
		return admission.NewForbidden(a, fmt.Errorf("user not permitted to approve requests with signerName %q", csr.Spec.SignerName))
	}

	return nil
}


// Validate verifies that the requesting user has permission to sign
// CertificateSigningRequests for the specified signerName.
func (p *Plugin) Validate(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) error {
	// Ignore all calls to anything other than 'certificatesigningrequests/status'.
	// Ignore all operations other than UPDATE.
	if a.GetSubresource() != "status" ||
		a.GetResource().GroupResource() != csrGroupResource {
		return nil
	}

	oldCSR, ok := a.GetOldObject().(*api.CertificateSigningRequest)
	if !ok {
		return admission.NewForbidden(a, fmt.Errorf("expected type CertificateSigningRequest, got: %T", a.GetOldObject()))
	}
	csr, ok := a.GetObject().(*api.CertificateSigningRequest)
	if !ok {
		return admission.NewForbidden(a, fmt.Errorf("expected type CertificateSigningRequest, got: %T", a.GetObject()))
	}

	// only run if the status.certificate or status.conditions field has been changed
	if reflect.DeepEqual(oldCSR.Status.Certificate, csr.Status.Certificate) && apiequality.Semantic.DeepEqual(oldCSR.Status.Conditions, csr.Status.Conditions) {
		return nil
	}

	if !certificates.IsAuthorizedForSignerName(ctx, p.authz, a.GetUserInfo(), "sign", oldCSR.Spec.SignerName) {
		klog.V(4).Infof("user not permitted to sign CertificateSigningRequest %q with signerName %q", oldCSR.Name, oldCSR.Spec.SignerName)
		return admission.NewForbidden(a, fmt.Errorf("user not permitted to sign requests with signerName %q", oldCSR.Spec.SignerName))
	}

	return nil
}


// Register registers the plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		return NewPlugin(), nil
	})
}


// ValidateInitialization always returns nil.
func (p *Plugin) ValidateInitialization() error {
	return nil
}


// NewPlugin constructs a new instance of the CertificateSubjectRestrictions admission interface.
func NewPlugin() *Plugin {
	return &Plugin{
		Handler: admission.NewHandler(admission.Create),
	}
}


// Validate ensures that if the signerName on a CSR is set to
// `kubernetes.io/kube-apiserver-client`, that its organization (group)
// attribute is not set to `system:masters`.
func (p *Plugin) Validate(_ context.Context, a admission.Attributes, _ admission.ObjectInterfaces) error {
	if a.GetResource().GroupResource() != csrGroupResource || a.GetSubresource() != "" {
		return nil
	}

	csr, ok := a.GetObject().(*certificatesapi.CertificateSigningRequest)
	if !ok {
		return admission.NewForbidden(a, fmt.Errorf("expected type CertificateSigningRequest, got: %T", a.GetObject()))
	}

	if csr.Spec.SignerName != certificatesv1beta1.KubeAPIServerClientSignerName {
		return nil
	}

	csrParsed, err := certificatesapi.ParseCSR(csr.Spec.Request)
	if err != nil {
		return admission.NewForbidden(a, fmt.Errorf("failed to parse CSR: %v", err))
	}

	for _, group := range csrParsed.Subject.Organization {
		if group == "system:masters" {
			klog.V(4).Infof("CSR %s rejected by admission plugin %s for attempting to use signer %s with system:masters group",
				csr.Name, PluginName, certificatesv1beta1.KubeAPIServerClientSignerName)
			return admission.NewForbidden(a, fmt.Errorf("use of %s signer with system:masters group is not allowed",
				certificatesv1beta1.KubeAPIServerClientSignerName))
		}
	}

	return nil
}


// IsAuthorizedForSignerName returns true if 'info' is authorized to perform the given
// 'verb' on the synthetic 'signers' resource with the given signerName.
// If the user does not have permission to perform the 'verb' on the given signerName,
// it will also perform an authorization check against {domain portion}/*, for example
// `kubernetes.io/*`. This allows an entity to be granted permission to 'verb' on all
// signerNames with a given 'domain portion'.
func IsAuthorizedForSignerName(ctx context.Context, authz authorizer.Authorizer, info user.Info, verb, signerName string) bool {
	// First check if the user has explicit permission to 'verb' for the given signerName.
	attr := buildAttributes(info, verb, signerName)
	decision, reason, err := authz.Authorize(ctx, attr)
	switch {
	case err != nil:
		klog.V(3).Infof("cannot authorize %q %q for policy: %v,%v", verb, attr.GetName(), reason, err)
	case decision == authorizer.DecisionAllow:
		return true
	}

	// If not, check if the user has wildcard permissions to 'verb' for the domain portion of the signerName, e.g.
	// 'kubernetes.io/*'.
	attr = buildWildcardAttributes(info, verb, signerName)
	decision, reason, err = authz.Authorize(ctx, attr)
	switch {
	case err != nil:
		klog.V(3).Infof("cannot authorize %q %q for policy: %v,%v", verb, attr.GetName(), reason, err)
	case decision == authorizer.DecisionAllow:
		return true
	}

	return false
}


func buildAttributes(info user.Info, verb, signerName string) authorizer.Attributes {
	return authorizer.AttributesRecord{
		User:            info,
		Verb:            verb,
		Name:            signerName,
		APIGroup:        "certificates.k8s.io",
		APIVersion:      "*",
		Resource:        "signers",
		ResourceRequest: true,
	}
}


func buildWildcardAttributes(info user.Info, verb, signerName string) authorizer.Attributes {
	parts := strings.Split(signerName, "/")
	domain := parts[0]
	return buildAttributes(info, verb, domain+"/*")
}


// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		return NewDefaultTolerationSeconds(), nil
	})
}


// NewDefaultTolerationSeconds creates a new instance of the DefaultTolerationSeconds admission controller
func NewDefaultTolerationSeconds() *Plugin {
	return &Plugin{
		Handler: admission.NewHandler(admission.Create, admission.Update),
	}
}


// Admit makes an admission decision based on the request attributes
func (p *Plugin) Admit(ctx context.Context, attributes admission.Attributes, o admission.ObjectInterfaces) (err error) {
	if attributes.GetResource().GroupResource() != api.Resource("pods") {
		return nil
	}

	if len(attributes.GetSubresource()) > 0 {
		// only run the checks below on pods proper and not subresources
		return nil
	}

	pod, ok := attributes.GetObject().(*api.Pod)
	if !ok {
		return errors.NewBadRequest(fmt.Sprintf("expected *api.Pod but got %T", attributes.GetObject()))
	}

	tolerations := pod.Spec.Tolerations

	toleratesNodeNotReady := false
	toleratesNodeUnreachable := false
	for _, toleration := range tolerations {
		if (toleration.Key == v1.TaintNodeNotReady || len(toleration.Key) == 0) &&
			(toleration.Effect == api.TaintEffectNoExecute || len(toleration.Effect) == 0) {
			toleratesNodeNotReady = true
		}

		if (toleration.Key == v1.TaintNodeUnreachable || len(toleration.Key) == 0) &&
			(toleration.Effect == api.TaintEffectNoExecute || len(toleration.Effect) == 0) {
			toleratesNodeUnreachable = true
		}
	}

	if !toleratesNodeNotReady {
		pod.Spec.Tolerations = append(pod.Spec.Tolerations, notReadyToleration)
	}

	if !toleratesNodeUnreachable {
		pod.Spec.Tolerations = append(pod.Spec.Tolerations, unreachableToleration)
	}

	return nil
}


// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		return NewAlwaysDeny(), nil
	})
}


// Admit makes an admission decision based on the request attributes.
func (alwaysDeny) Admit(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) (err error) {
	return admission.NewForbidden(a, errors.New("admission control is denying all modifications"))
}


// Validate makes an admission decision based on the request attributes.  It is NOT allowed to mutate.
func (alwaysDeny) Validate(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) (err error) {
	return admission.NewForbidden(a, errors.New("admission control is denying all modifications"))
}


// Handles returns true if this admission controller can handle the given operation
// where operation can be one of CREATE, UPDATE, DELETE, or CONNECT
func (alwaysDeny) Handles(operation admission.Operation) bool {
	return true
}


// NewAlwaysDeny creates an always deny admission handler
func NewAlwaysDeny() admission.Interface {
	// DEPRECATED: AlwaysDeny denys all admission request, it is no use.
	klog.Warningf("%s admission controller is deprecated. "+
		"Please remove this controller from your configuration files and scripts.", PluginName)
	return new(alwaysDeny)
}


// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName,
		func(config io.Reader) (admission.Interface, error) {
			// load the configuration provided (if any)
			configuration, err := LoadConfiguration(config)
			if err != nil {
				return nil, err
			}
			// validate the configuration (if any)
			if configuration != nil {
				if errs := validation.ValidateConfiguration(configuration); len(errs) != 0 {
					return nil, errs.ToAggregate()
				}
			}
			return newEventRateLimit(configuration, clock.RealClock{})
		})
}


// newEventRateLimit configures an admission controller that can enforce event rate limits
func newEventRateLimit(config *eventratelimitapi.Configuration, clock flowcontrol.Clock) (*Plugin, error) {
	limitEnforcers := make([]*limitEnforcer, 0, len(config.Limits))
	for _, limitConfig := range config.Limits {
		enforcer, err := newLimitEnforcer(limitConfig, clock)
		if err != nil {
			return nil, err
		}
		limitEnforcers = append(limitEnforcers, enforcer)
	}

	eventRateLimitAdmission := &Plugin{
		Handler:        admission.NewHandler(admission.Create, admission.Update),
		limitEnforcers: limitEnforcers,
	}

	return eventRateLimitAdmission, nil
}


// Validate makes admission decisions while enforcing event rate limits
func (a *Plugin) Validate(ctx context.Context, attr admission.Attributes, o admission.ObjectInterfaces) (err error) {
	// ignore all operations that do not correspond to an Event kind
	if attr.GetKind().GroupKind() != api.Kind("Event") {
		return nil
	}

	// ignore all requests that specify dry-run
	// because they don't correspond to any calls to etcd,
	// they should not be affected by the ratelimit
	if attr.IsDryRun() {
		return nil
	}

	var errors []error
	// give each limit enforcer a chance to reject the event
	for _, enforcer := range a.limitEnforcers {
		if err := enforcer.accept(attr); err != nil {
			errors = append(errors, err)
		}
	}

	if aggregatedErr := utilerrors.NewAggregate(errors); aggregatedErr != nil {
		return apierrors.NewTooManyRequestsError(aggregatedErr.Error())
	}

	return nil
}


func Install(scheme *runtime.Scheme)

// Install registers the API group and adds types to a scheme
func Install(scheme *runtime.Scheme) {
	utilruntime.Must(internalapi.AddToScheme(scheme))
	utilruntime.Must(versionedapi.AddToScheme(scheme))
	utilruntime.Must(scheme.SetVersionPriority(versionedapi.SchemeGroupVersion))
}


func Kind(kind string) schema.GroupKind

// Kind takes an unqualified kind and returns a Group qualified GroupKind
func Kind(kind string) schema.GroupKind {
	return SchemeGroupVersion.WithKind(kind).GroupKind()
}


// Resource takes an unqualified resource and returns a Group qualified GroupResource
func Resource(resource string) schema.GroupResource {
	return SchemeGroupVersion.WithResource(resource).GroupResource()
}


func addKnownTypes(scheme *runtime.Scheme) error {
	// TODO this will get cleaned up with the scheme types are fixed
	scheme.AddKnownTypes(SchemeGroupVersion,
		&Configuration{},
	)
	return nil
}


func addDefaultingFuncs(scheme *kruntime.Scheme) error {
	return RegisterDefaults(scheme)
}


func SetDefaults_Configuration(obj *Configuration) {}


func init()

func init() {
	// We only register manually written functions here. The registration of the
	// generated functions takes place in the generated files. The separation
	// makes the code compile even when the generated files are missing.
	localSchemeBuilder.Register(addKnownTypes, addDefaultingFuncs)
}


func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&Configuration{},
	)
	return nil
}


// ValidateConfiguration validates the configuration.
func ValidateConfiguration(config *eventratelimitapi.Configuration) field.ErrorList {
	allErrs := field.ErrorList{}
	limitsPath := field.NewPath("limits")
	if len(config.Limits) == 0 {
		allErrs = append(allErrs, field.Invalid(limitsPath, config.Limits, "must not be empty"))
	}
	for i, limit := range config.Limits {
		idxPath := limitsPath.Index(i)
		if !limitTypes[limit.Type] {
			allowedValues := make([]string, len(limitTypes))
			i := 0
			for limitType := range limitTypes {
				allowedValues[i] = string(limitType)
				i++
			}
			allErrs = append(allErrs, field.NotSupported(idxPath.Child("type"), limit.Type, allowedValues))
		}
		if limit.Burst <= 0 {
			allErrs = append(allErrs, field.Invalid(idxPath.Child("burst"), limit.Burst, "must be positive"))
		}
		if limit.QPS <= 0 {
			allErrs = append(allErrs, field.Invalid(idxPath.Child("qps"), limit.QPS, "must be positive"))
		}
		if limit.Type != eventratelimitapi.ServerLimitType {
			if limit.CacheSize < 0 {
				allErrs = append(allErrs, field.Invalid(idxPath.Child("cacheSize"), limit.CacheSize, "must not be negative"))
			}
		}
	}
	return allErrs
}


func (c *singleCache) get(key interface{}) flowcontrol.RateLimiter {
	return c.rateLimiter
}


func (c *lruCache) get(key interface{}) flowcontrol.RateLimiter {
	value, found := c.cache.Get(key)
	if !found {
		rateLimter := c.rateLimiterFactory()
		c.cache.Add(key, rateLimter)
		return rateLimter
	}
	return value.(flowcontrol.RateLimiter)
}


func init() {
	install.Install(scheme)
}


// LoadConfiguration loads the provided configuration.
func LoadConfiguration(config io.Reader) (*eventratelimitapi.Configuration, error) {
	// if no config is provided, return a default configuration
	if config == nil {
		externalConfig := &eventratelimitv1alpha1.Configuration{}
		scheme.Default(externalConfig)
		internalConfig := &eventratelimitapi.Configuration{}
		if err := scheme.Convert(externalConfig, internalConfig, nil); err != nil {
			return nil, err
		}
		return internalConfig, nil
	}
	// we have a config so parse it.
	data, err := ioutil.ReadAll(config)
	if err != nil {
		return nil, err
	}
	decoder := codecs.UniversalDecoder()
	decodedObj, err := runtime.Decode(decoder, data)
	if err != nil {
		return nil, err
	}
	resourceQuotaConfiguration, ok := decodedObj.(*eventratelimitapi.Configuration)
	if !ok {
		return nil, fmt.Errorf("unexpected type: %T", decodedObj)
	}
	return resourceQuotaConfiguration, nil
}


func newLimitEnforcer(config eventratelimitapi.Limit, clock flowcontrol.Clock) (*limitEnforcer, error) {
	rateLimiterFactory := func() flowcontrol.RateLimiter {
		return flowcontrol.NewTokenBucketRateLimiterWithClock(float32(config.QPS), int(config.Burst), clock)
	}

	if config.Type == eventratelimitapi.ServerLimitType {
		return &limitEnforcer{
			limitType: config.Type,
			cache: &singleCache{
				rateLimiter: rateLimiterFactory(),
			},
			keyFunc: getServerKey,
		}, nil
	}

	cacheSize := int(config.CacheSize)
	if cacheSize == 0 {
		cacheSize = defaultCacheSize
	}
	underlyingCache := lru.New(cacheSize)
	cache := &lruCache{
		rateLimiterFactory: rateLimiterFactory,
		cache:              underlyingCache,
	}

	var keyFunc func(admission.Attributes) string
	switch t := config.Type; t {
	case eventratelimitapi.NamespaceLimitType:
		keyFunc = getNamespaceKey
	case eventratelimitapi.UserLimitType:
		keyFunc = getUserKey
	case eventratelimitapi.SourceAndObjectLimitType:
		keyFunc = getSourceAndObjectKey
	default:
		return nil, fmt.Errorf("unknown event rate limit type: %v", t)
	}

	return &limitEnforcer{
		limitType: config.Type,
		cache:     cache,
		keyFunc:   keyFunc,
	}, nil
}