readDirNames

// readDirNames reads the directory named by dirname and returns
// a list of directory entries.
// We are not using filepath.readDirNames because we do not want to sort files found in a directory before changing
// permissions for performance reasons.
func readDirNames(dirname string) ([]string, error) {
	f, err := os.Open(dirname)
	if err != nil {
		return nil, err
	}
	names, err := f.Readdirnames(-1)
	f.Close()
	if err != nil {
		return nil, err
	}
	return names, nil
}


walkDeep

// walkDeep can be used to traverse directories and has two minor differences
// from filepath.Walk:
//   - List of files/dirs is not sorted for performance reasons
//   - callback walkFunc is invoked on root directory after visiting children dirs and files
func walkDeep(root string, walkFunc filepath.WalkFunc) error {
	info, err := os.Lstat(root)
	if err != nil {
		return walkFunc(root, nil, err)
	}
	return walk(root, info, walkFunc)
}


walk

func walk(path string, info os.FileInfo, walkFunc filepath.WalkFunc) error {
	if !info.IsDir() {
		return walkFunc(path, info, nil)
	}
	names, err := readDirNames(path)
	if err != nil {
		return err
	}
	for _, name := range names {
		filename := filepath.Join(path, name)
		fileInfo, err := os.Lstat(filename)
		if err != nil {
			if err := walkFunc(filename, fileInfo, err); err != nil {
				return err
			}
		} else {
			err = walk(filename, fileInfo, walkFunc)
			if err != nil {
				return err
			}
		}
	}
	return walkFunc(path, info, nil)
}


SetVolumeOwnership

func SetVolumeOwnership(mounter Mounter, dir string, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy, completeFunc func(types.CompleteFuncParam)) error {
	return nil
}


NewAttacher

func (plugin *vsphereVolumePlugin) NewAttacher() (volume.Attacher, error) {
	vsphereCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
	if err != nil {
		return nil, err
	}

	return &vsphereVMDKAttacher{
		host:           plugin.host,
		vsphereVolumes: vsphereCloud,
	}, nil
}


NewDeviceMounter

func (plugin *vsphereVolumePlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
	return plugin.NewAttacher()
}


Attach

// Attaches the volume specified by the given spec to the given host.
// On success, returns the device path where the device was attached on the
// node.
// Callers are responsible for retryinging on failure.
// Callers are responsible for thread safety between concurrent attach and
// detach operations.
func (attacher *vsphereVMDKAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	klog.V(4).Infof("vSphere: Attach disk called for node %s", nodeName)

	// Keeps concurrent attach operations to same host atomic
	attachdetachMutex.LockKey(string(nodeName))
	defer attachdetachMutex.UnlockKey(string(nodeName))

	// vsphereCloud.AttachDisk checks if disk is already attached to host and
	// succeeds in that case, so no need to do that separately.
	diskUUID, err := attacher.vsphereVolumes.AttachDisk(volumeSource.VolumePath, volumeSource.StoragePolicyName, nodeName)
	if err != nil {
		klog.Errorf("Error attaching volume %q to node %q: %+v", volumeSource.VolumePath, nodeName, err)
		return "", err
	}

	return filepath.Join(diskByIDPath, diskSCSIPrefix+diskUUID), nil
}


VolumesAreAttached

func (attacher *vsphereVMDKAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.NodeName) (map[*volume.Spec]bool, error) {
	klog.Warningf("Attacher.VolumesAreAttached called for node %q - Please use BulkVerifyVolumes for vSphere", nodeName)
	volumeNodeMap := map[types.NodeName][]*volume.Spec{
		nodeName: specs,
	}
	nodeVolumesResult := make(map[*volume.Spec]bool)
	nodesVerificationMap, err := attacher.BulkVerifyVolumes(volumeNodeMap)
	if err != nil {
		klog.Errorf("Attacher.VolumesAreAttached - error checking volumes for node %q with %v", nodeName, err)
		return nodeVolumesResult, err
	}
	if result, ok := nodesVerificationMap[nodeName]; ok {
		return result, nil
	}
	return nodeVolumesResult, nil
}


BulkVerifyVolumes

func (attacher *vsphereVMDKAttacher) BulkVerifyVolumes(volumesByNode map[types.NodeName][]*volume.Spec) (map[types.NodeName]map[*volume.Spec]bool, error) {
	volumesAttachedCheck := make(map[types.NodeName]map[*volume.Spec]bool)
	volumePathsByNode := make(map[types.NodeName][]string)
	volumeSpecMap := make(map[string]*volume.Spec)

	for nodeName, volumeSpecs := range volumesByNode {
		for _, volumeSpec := range volumeSpecs {
			volumeSource, _, err := getVolumeSource(volumeSpec)
			if err != nil {
				klog.Errorf("Error getting volume (%q) source : %v", volumeSpec.Name(), err)
				continue
			}
			volPath := volumeSource.VolumePath
			volumePathsByNode[nodeName] = append(volumePathsByNode[nodeName], volPath)
			nodeVolume, nodeVolumeExists := volumesAttachedCheck[nodeName]
			if !nodeVolumeExists {
				nodeVolume = make(map[*volume.Spec]bool)
			}
			nodeVolume[volumeSpec] = true
			volumeSpecMap[volPath] = volumeSpec
			volumesAttachedCheck[nodeName] = nodeVolume
		}
	}
	attachedResult, err := attacher.vsphereVolumes.DisksAreAttached(volumePathsByNode)
	if err != nil {
		klog.Errorf("Error checking if volumes are attached to nodes: %+v. err: %v", volumePathsByNode, err)
		return volumesAttachedCheck, err
	}

	for nodeName, nodeVolumes := range attachedResult {
		for volumePath, attached := range nodeVolumes {
			if !attached {
				spec := volumeSpecMap[volumePath]
				setNodeVolume(volumesAttachedCheck, spec, nodeName, false)
			}
		}
	}
	return volumesAttachedCheck, nil
}


WaitForAttach

func (attacher *vsphereVMDKAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	if devicePath == "" {
		return "", fmt.Errorf("WaitForAttach failed for VMDK %q: devicePath is empty", volumeSource.VolumePath)
	}

	ticker := time.NewTicker(checkSleepDuration)
	defer ticker.Stop()

	timer := time.NewTimer(timeout)
	defer timer.Stop()

	for {
		select {
		case <-ticker.C:
			klog.V(5).Infof("Checking VMDK %q is attached", volumeSource.VolumePath)
			path, err := verifyDevicePath(devicePath)
			if err != nil {
				// Log error, if any, and continue checking periodically. See issue #11321
				klog.Warningf("Error verifying VMDK (%q) is attached: %v", volumeSource.VolumePath, err)
			} else if path != "" {
				// A device path has successfully been created for the VMDK
				klog.Infof("Successfully found attached VMDK %q.", volumeSource.VolumePath)
				return path, nil
			}
		case <-timer.C:
			return "", fmt.Errorf("could not find attached VMDK %q. Timeout waiting for mount paths to be created", volumeSource.VolumePath)
		}
	}
}


GetDeviceMountPath

// GetDeviceMountPath returns a path where the device should
// point which should be bind mounted for individual volumes.
func (attacher *vsphereVMDKAttacher) GetDeviceMountPath(spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return makeGlobalPDPath(attacher.host, volumeSource.VolumePath), nil
}


GetDeviceMountRefs

// GetMountDeviceRefs finds all other references to the device referenced
// by deviceMountPath; returns a list of paths.
func (plugin *vsphereVolumePlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
	mounter := plugin.host.GetMounter(plugin.GetPluginName())
	return mounter.GetMountRefs(deviceMountPath)
}


MountDevice

// MountDevice mounts device to global mount point.
func (attacher *vsphereVMDKAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string, _ volume.DeviceMounterArgs) error {
	klog.Infof("vsphere MountDevice mount %s to %s", devicePath, deviceMountPath)
	mounter := attacher.host.GetMounter(vsphereVolumePluginName)
	notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath)
	if err != nil {
		if os.IsNotExist(err) {
			dir := deviceMountPath
			if runtime.GOOS == "windows" {
				dir = filepath.Dir(deviceMountPath)
			}
			if err := os.MkdirAll(dir, 0750); err != nil {
				klog.Errorf("Failed to create directory at %#v. err: %s", dir, err)
				return err
			}
			notMnt = true
		} else {
			return err
		}
	}

	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return err
	}

	options := []string{}

	if notMnt {
		diskMounter := volumeutil.NewSafeFormatAndMountFromHost(vsphereVolumePluginName, attacher.host)
		mountOptions := volumeutil.MountOptionFromSpec(spec, options...)
		err = diskMounter.FormatAndMount(devicePath, deviceMountPath, volumeSource.FSType, mountOptions)
		if err != nil {
			os.Remove(deviceMountPath)
			return err
		}
		klog.V(4).Infof("formatting spec %v devicePath %v deviceMountPath %v fs %v with options %+v", spec.Name(), devicePath, deviceMountPath, volumeSource.FSType, options)
	}
	return nil
}


NewDetacher

func (plugin *vsphereVolumePlugin) NewDetacher() (volume.Detacher, error) {
	vsphereCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
	if err != nil {
		return nil, err
	}

	return &vsphereVMDKDetacher{
		mounter:        plugin.host.GetMounter(plugin.GetPluginName()),
		vsphereVolumes: vsphereCloud,
	}, nil
}


NewDeviceUnmounter

func (plugin *vsphereVolumePlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
	return plugin.NewDetacher()
}


Detach

// Detach the given device from the given node.
func (detacher *vsphereVMDKDetacher) Detach(volumeName string, nodeName types.NodeName) error {
	volPath := getVolPathfromVolumeName(volumeName)
	attached, newVolumePath, err := detacher.vsphereVolumes.DiskIsAttached(volPath, nodeName)
	if err != nil {
		// Log error and continue with detach
		klog.Errorf(
			"Error checking if volume (%q) is already attached to current node (%q). Will continue and try detach anyway. err=%v",
			volPath, nodeName, err)
	}

	if err == nil && !attached {
		// Volume is already detached from node.
		klog.Infof("detach operation was successful. volume %q is already detached from node %q.", volPath, nodeName)
		return nil
	}

	attachdetachMutex.LockKey(string(nodeName))
	defer attachdetachMutex.UnlockKey(string(nodeName))
	if err := detacher.vsphereVolumes.DetachDisk(newVolumePath, nodeName); err != nil {
		klog.Errorf("Error detaching volume %q: %v", volPath, err)
		return err
	}
	return nil
}


UnmountDevice

func (detacher *vsphereVMDKDetacher) UnmountDevice(deviceMountPath string) error {
	return mount.CleanupMountPoint(deviceMountPath, detacher.mounter, false)
}


CanAttach

func (plugin *vsphereVolumePlugin) CanAttach(spec *volume.Spec) (bool, error) {
	return true, nil
}


CanDeviceMount

func (plugin *vsphereVolumePlugin) CanDeviceMount(spec *volume.Spec) (bool, error) {
	return true, nil
}


setNodeVolume

func setNodeVolume(
	nodeVolumeMap map[types.NodeName]map[*volume.Spec]bool,
	volumeSpec *volume.Spec,
	nodeName types.NodeName,
	check bool) {

	volumeMap := nodeVolumeMap[nodeName]
	if volumeMap == nil {
		volumeMap = make(map[*volume.Spec]bool)
		nodeVolumeMap[nodeName] = volumeMap
	}
	volumeMap[volumeSpec] = check
}


ProbeVolumePlugins

// This is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&vsphereVolumePlugin{}}
}


getPath

func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
	return host.GetPodVolumeDir(uid, utilstrings.EscapeQualifiedName(vsphereVolumePluginName), volName)
}


Init

// vSphere Volume Plugin
func (plugin *vsphereVolumePlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	return nil
}


GetPluginName

func (plugin *vsphereVolumePlugin) GetPluginName() string {
	return vsphereVolumePluginName
}


IsMigratedToCSI

func (plugin *vsphereVolumePlugin) IsMigratedToCSI() bool {
	return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationvSphere)
}


GetVolumeName

func (plugin *vsphereVolumePlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return volumeSource.VolumePath, nil
}


CanSupport

func (plugin *vsphereVolumePlugin) CanSupport(spec *volume.Spec) bool {
	return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.VsphereVolume != nil) ||
		(spec.Volume != nil && spec.Volume.VsphereVolume != nil)
}


RequiresRemount

func (plugin *vsphereVolumePlugin) RequiresRemount(spec *volume.Spec) bool {
	return false
}


SupportsMountOption

func (plugin *vsphereVolumePlugin) SupportsMountOption() bool {
	return true
}


SupportsBulkVolumeVerification

func (plugin *vsphereVolumePlugin) SupportsBulkVolumeVerification() bool {
	return true
}


SupportsSELinuxContextMount

func (plugin *vsphereVolumePlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return false, nil
}


NewMounter

func (plugin *vsphereVolumePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
	return plugin.newMounterInternal(spec, pod.UID, &VsphereDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName()))
}


NewUnmounter

func (plugin *vsphereVolumePlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	return plugin.newUnmounterInternal(volName, podUID, &VsphereDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName()))
}


newMounterInternal

func (plugin *vsphereVolumePlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager vdManager, mounter mount.Interface) (volume.Mounter, error) {
	vvol, _, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}

	volPath := vvol.VolumePath
	fsType := vvol.FSType

	return &vsphereVolumeMounter{
		vsphereVolume: &vsphereVolume{
			podUID:          podUID,
			volName:         spec.Name(),
			volPath:         volPath,
			manager:         manager,
			mounter:         mounter,
			plugin:          plugin,
			MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, spec.Name(), plugin.host)),
		},
		fsType:       fsType,
		diskMounter:  util.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host),
		mountOptions: util.MountOptionFromSpec(spec),
	}, nil
}


newUnmounterInternal

func (plugin *vsphereVolumePlugin) newUnmounterInternal(volName string, podUID types.UID, manager vdManager, mounter mount.Interface) (volume.Unmounter, error) {
	return &vsphereVolumeUnmounter{
		&vsphereVolume{
			podUID:          podUID,
			volName:         volName,
			manager:         manager,
			mounter:         mounter,
			plugin:          plugin,
			MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)),
		}}, nil
}


ConstructVolumeSpec

func (plugin *vsphereVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
	mounter := plugin.host.GetMounter(plugin.GetPluginName())
	kvh, ok := plugin.host.(volume.KubeletVolumeHost)
	if !ok {
		return volume.ReconstructedVolume{}, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
	}
	hu := kvh.GetHostUtil()
	pluginMntDir := util.GetPluginMountDir(plugin.host, plugin.GetPluginName())
	volumePath, err := hu.GetDeviceNameFromMount(mounter, mountPath, pluginMntDir)
	if err != nil {
		return volume.ReconstructedVolume{}, err
	}
	volumePath = strings.Replace(volumePath, "\\040", " ", -1)
	klog.V(5).Infof("vSphere volume path is %q", volumePath)
	vsphereVolume := &v1.Volume{
		Name: volumeName,
		VolumeSource: v1.VolumeSource{
			VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
				VolumePath: volumePath,
			},
		},
	}
	return volume.ReconstructedVolume{
		Spec: volume.NewSpecFromVolume(vsphereVolume),
	}, nil
}


GetAttributes

func (b *vsphereVolumeMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		SELinuxRelabel: true,
		Managed:        true,
	}
}


SetUp

// SetUp attaches the disk and bind mounts to the volume path.
func (b *vsphereVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error {
	return b.SetUpAt(b.GetPath(), mounterArgs)
}


SetUpAt

// SetUp attaches the disk and bind mounts to the volume path.
func (b *vsphereVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	klog.V(5).Infof("vSphere volume setup %s to %s", b.volPath, dir)

	// TODO: handle failed mounts here.
	notmnt, err := b.mounter.IsLikelyNotMountPoint(dir)
	if err != nil && !os.IsNotExist(err) {
		klog.V(4).Infof("IsLikelyNotMountPoint failed: %v", err)
		return err
	}
	if !notmnt {
		klog.V(4).Infof("Something is already mounted to target %s", dir)
		return nil
	}

	if runtime.GOOS != "windows" {
		// On Windows, Mount will create the parent of dir and mklink (create a symbolic link) at dir later, so don't create a
		// directory at dir now. Otherwise mklink will error: "Cannot create a file when that file already exists".
		if err := os.MkdirAll(dir, 0750); err != nil {
			klog.Errorf("Could not create directory %s: %v", dir, err)
			return err
		}
	}

	options := []string{"bind"}

	// Perform a bind mount to the full path to allow duplicate mounts of the same PD.
	globalPDPath := makeGlobalPDPath(b.plugin.host, b.volPath)
	mountOptions := util.JoinMountOptions(options, b.mountOptions)
	err = b.mounter.MountSensitiveWithoutSystemd(globalPDPath, dir, "", mountOptions, nil)
	if err != nil {
		notmnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir)
		if mntErr != nil {
			klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
			return err
		}
		if !notmnt {
			if mntErr = b.mounter.Unmount(dir); mntErr != nil {
				klog.Errorf("Failed to unmount: %v", mntErr)
				return err
			}
			notmnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir)
			if mntErr != nil {
				klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
				return err
			}
			if !notmnt {
				klog.Errorf("%s is still mounted, despite call to unmount().  Will try again next sync loop.", b.GetPath())
				return err
			}
		}
		os.Remove(dir)
		return err
	}
	volume.SetVolumeOwnership(b, dir, mounterArgs.FsGroup, mounterArgs.FSGroupChangePolicy, util.FSGroupCompleteHook(b.plugin, nil))
	klog.V(3).Infof("vSphere volume %s mounted to %s", b.volPath, dir)

	return nil
}


TearDown

// Unmounts the bind mount, and detaches the disk only if the PD
// resource was the last reference to that disk on the kubelet.
func (v *vsphereVolumeUnmounter) TearDown() error {
	return v.TearDownAt(v.GetPath())
}


TearDownAt

// Unmounts the bind mount, and detaches the disk only if the PD
// resource was the last reference to that disk on the kubelet.
func (v *vsphereVolumeUnmounter) TearDownAt(dir string) error {
	return mount.CleanupMountPoint(dir, v.mounter, false)
}


makeGlobalPDPath

func makeGlobalPDPath(host volume.VolumeHost, devName string) string {
	return filepath.Join(host.GetPluginDir(vsphereVolumePluginName), util.MountsInGlobalPDPath, devName)
}


GetPath

func (vv *vsphereVolume) GetPath() string {
	name := vsphereVolumePluginName
	return vv.plugin.host.GetPodVolumeDir(vv.podUID, utilstrings.EscapeQualifiedName(name), vv.volName)
}


GetAccessModes

// vSphere Persistent Volume Plugin
func (plugin *vsphereVolumePlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
	return []v1.PersistentVolumeAccessMode{
		v1.ReadWriteOnce,
	}
}


NewDeleter

func (plugin *vsphereVolumePlugin) NewDeleter(logger klog.Logger, spec *volume.Spec) (volume.Deleter, error) {
	return plugin.newDeleterInternal(spec, &VsphereDiskUtil{})
}


newDeleterInternal

func (plugin *vsphereVolumePlugin) newDeleterInternal(spec *volume.Spec, manager vdManager) (volume.Deleter, error) {
	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.VsphereVolume == nil {
		return nil, fmt.Errorf("spec.PersistentVolumeSource.VsphereVolume is nil")
	}
	return &vsphereVolumeDeleter{
		&vsphereVolume{
			volName: spec.Name(),
			volPath: spec.PersistentVolume.Spec.VsphereVolume.VolumePath,
			manager: manager,
			plugin:  plugin,
		}}, nil
}


Delete

func (r *vsphereVolumeDeleter) Delete() error {
	return r.manager.DeleteVolume(r)
}


NewProvisioner

func (plugin *vsphereVolumePlugin) NewProvisioner(logger klog.Logger, options volume.VolumeOptions) (volume.Provisioner, error) {
	return plugin.newProvisionerInternal(options, &VsphereDiskUtil{})
}


newProvisionerInternal

func (plugin *vsphereVolumePlugin) newProvisionerInternal(options volume.VolumeOptions, manager vdManager) (volume.Provisioner, error) {
	return &vsphereVolumeProvisioner{
		vsphereVolume: &vsphereVolume{
			manager: manager,
			plugin:  plugin,
		},
		options: options,
	}, nil
}


Provision

func (v *vsphereVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTopologies []v1.TopologySelectorTerm) (*v1.PersistentVolume, error) {
	if !util.ContainsAllAccessModes(v.plugin.GetAccessModes(), v.options.PVC.Spec.AccessModes) {
		return nil, fmt.Errorf("invalid AccessModes %v: only AccessModes %v are supported", v.options.PVC.Spec.AccessModes, v.plugin.GetAccessModes())
	}
	klog.V(1).Infof("Provision with selectedNode: %s and allowedTopologies : %s", getNodeName(selectedNode), allowedTopologies)
	selectedZones, err := volumehelpers.ZonesFromAllowedTopologies(allowedTopologies)
	if err != nil {
		return nil, err
	}

	klog.V(4).Infof("Selected zones for volume : %s", selectedZones)
	volSpec, err := v.manager.CreateVolume(v, selectedNode, selectedZones.List())
	if err != nil {
		return nil, err
	}

	if volSpec.Fstype == "" {
		volSpec.Fstype = "ext4"
	}

	volumeMode := v.options.PVC.Spec.VolumeMode
	if volumeMode != nil && *volumeMode == v1.PersistentVolumeBlock {
		klog.V(5).Infof("vSphere block volume should not have any FSType")
		volSpec.Fstype = ""
	}

	pv := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name:   v.options.PVName,
			Labels: map[string]string{},
			Annotations: map[string]string{
				util.VolumeDynamicallyCreatedByKey: "vsphere-volume-dynamic-provisioner",
			},
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeReclaimPolicy: v.options.PersistentVolumeReclaimPolicy,
			AccessModes:                   v.options.PVC.Spec.AccessModes,
			Capacity: v1.ResourceList{
				v1.ResourceName(v1.ResourceStorage): resource.MustParse(fmt.Sprintf("%dKi", volSpec.Size)),
			},
			VolumeMode: volumeMode,
			PersistentVolumeSource: v1.PersistentVolumeSource{
				VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
					VolumePath:        volSpec.Path,
					FSType:            volSpec.Fstype,
					StoragePolicyName: volSpec.StoragePolicyName,
					StoragePolicyID:   volSpec.StoragePolicyID,
				},
			},
			MountOptions: v.options.MountOptions,
		},
	}
	if len(v.options.PVC.Spec.AccessModes) == 0 {
		pv.Spec.AccessModes = v.plugin.GetAccessModes()
	}

	labels := volSpec.Labels
	requirements := make([]v1.NodeSelectorRequirement, 0)
	if len(labels) != 0 {
		if pv.Labels == nil {
			pv.Labels = make(map[string]string)
		}
		for k, v := range labels {
			pv.Labels[k] = v
			var values []string
			if k == v1.LabelTopologyZone || k == v1.LabelFailureDomainBetaZone {
				values, err = volumehelpers.LabelZonesToList(v)
				if err != nil {
					return nil, fmt.Errorf("failed to convert label string for Zone: %s to a List: %v", v, err)
				}
			} else {
				values = []string{v}
			}
			requirements = append(requirements, v1.NodeSelectorRequirement{Key: k, Operator: v1.NodeSelectorOpIn, Values: values})
		}
	}

	if len(requirements) > 0 {
		pv.Spec.NodeAffinity = new(v1.VolumeNodeAffinity)
		pv.Spec.NodeAffinity.Required = new(v1.NodeSelector)
		pv.Spec.NodeAffinity.Required.NodeSelectorTerms = make([]v1.NodeSelectorTerm, 1)
		pv.Spec.NodeAffinity.Required.NodeSelectorTerms[0].MatchExpressions = requirements
	}

	return pv, nil
}


getVolumeSource

func getVolumeSource(
	spec *volume.Spec) (*v1.VsphereVirtualDiskVolumeSource, bool, error) {
	if spec.Volume != nil && spec.Volume.VsphereVolume != nil {
		return spec.Volume.VsphereVolume, spec.ReadOnly, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.VsphereVolume != nil {
		return spec.PersistentVolume.Spec.VsphereVolume, spec.ReadOnly, nil
	}

	return nil, false, fmt.Errorf("spec does not reference a VSphere volume type")
}


getNodeName

func getNodeName(node *v1.Node) string {
	if node == nil {
		return ""
	}
	return node.Name
}


ConstructBlockVolumeSpec

func (plugin *vsphereVolumePlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mapPath string) (*volume.Spec, error) {

	pluginDir := plugin.host.GetPluginDir(plugin.GetPluginName())
	blkUtil := volumepathhandler.NewBlockVolumePathHandler()
	globalMapPathUUID, err := blkUtil.FindGlobalMapPathUUIDFromPod(pluginDir, mapPath, podUID)
	if err != nil {
		klog.Errorf("Failed to find GlobalMapPathUUID from Pod: %s with error: %+v", podUID, err)
		return nil, err
	}
	klog.V(5).Infof("globalMapPathUUID: %v", globalMapPathUUID)
	globalMapPath := filepath.Dir(globalMapPathUUID)
	if len(globalMapPath) <= 1 {
		return nil, fmt.Errorf("failed to get volume plugin information from globalMapPathUUID: %v", globalMapPathUUID)
	}
	return getVolumeSpecFromGlobalMapPath(volumeName, globalMapPath)
}


getVolumeSpecFromGlobalMapPath

func getVolumeSpecFromGlobalMapPath(volumeName, globalMapPath string) (*volume.Spec, error) {
	// Construct volume spec from globalMapPath
	// globalMapPath example:
	//   plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumeID}
	//   plugins/kubernetes.io/vsphere-volume/volumeDevices/[datastore1]\\040volumes/myDisk
	volPath := filepath.Base(globalMapPath)
	volPath = strings.Replace(volPath, "\\040", "", -1)
	if len(volPath) <= 1 {
		return nil, fmt.Errorf("failed to get volume path from global path=%s", globalMapPath)
	}
	block := v1.PersistentVolumeBlock
	vsphereVolume := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: volumeName,
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeSource: v1.PersistentVolumeSource{
				VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
					VolumePath: volPath,
				},
			},
			VolumeMode: &block,
		},
	}
	return volume.NewSpecFromPersistentVolume(vsphereVolume, true), nil
}


NewBlockVolumeMapper

func (plugin *vsphereVolumePlugin) NewBlockVolumeMapper(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.BlockVolumeMapper, error) {
	// If this called via GenerateUnmapDeviceFunc(), pod is nil.
	// Pass empty string as dummy uid since uid isn't used in the case.
	var uid types.UID
	if pod != nil {
		uid = pod.UID
	}
	return plugin.newBlockVolumeMapperInternal(spec, uid, &VsphereDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName()))
}


newBlockVolumeMapperInternal

func (plugin *vsphereVolumePlugin) newBlockVolumeMapperInternal(spec *volume.Spec, podUID types.UID, manager vdManager, mounter mount.Interface) (volume.BlockVolumeMapper, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		klog.Errorf("Failed to get Volume source from volume Spec: %+v with error: %+v", *spec, err)
		return nil, err
	}
	volPath := volumeSource.VolumePath
	mapper := &vsphereBlockVolumeMapper{
		vsphereVolume: &vsphereVolume{
			volName:         spec.Name(),
			podUID:          podUID,
			volPath:         volPath,
			manager:         manager,
			mounter:         mounter,
			plugin:          plugin,
			MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, spec.Name(), plugin.host)),
		},
	}

	blockPath, err := mapper.GetGlobalMapPath(spec)
	if err != nil {
		return nil, fmt.Errorf("failed to get device path: %v", err)
	}
	mapper.MetricsProvider = volume.NewMetricsBlock(filepath.Join(blockPath, string(podUID)))

	return mapper, nil
}


NewBlockVolumeUnmapper

func (plugin *vsphereVolumePlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (volume.BlockVolumeUnmapper, error) {
	return plugin.newUnmapperInternal(volName, podUID, &VsphereDiskUtil{})
}


newUnmapperInternal

func (plugin *vsphereVolumePlugin) newUnmapperInternal(volName string, podUID types.UID, manager vdManager) (volume.BlockVolumeUnmapper, error) {
	return &vsphereBlockVolumeUnmapper{
		vsphereVolume: &vsphereVolume{
			volName: volName,
			podUID:  podUID,
			volPath: volName,
			manager: manager,
			plugin:  plugin,
		},
	}, nil
}


GetGlobalMapPath

// GetGlobalMapPath returns global map path and error
// path: plugins/kubernetes.io/{PluginName}/volumeDevices/volumePath
func (v *vsphereVolume) GetGlobalMapPath(spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}
	return filepath.Join(v.plugin.host.GetVolumeDevicePluginDir(vsphereVolumePluginName), string(volumeSource.VolumePath)), nil
}


GetPodDeviceMapPath

func (v *vsphereVolume) GetPodDeviceMapPath() (string, string) {
	return v.plugin.host.GetPodVolumeDeviceDir(v.podUID, utilstrings.EscapeQualifiedName(vsphereVolumePluginName)), v.volName
}


SupportsMetrics

// SupportsMetrics returns true for vsphereBlockVolumeMapper as it initializes the
// MetricsProvider.
func (vbvm *vsphereBlockVolumeMapper) SupportsMetrics() bool {
	return true
}


CreateVolume

// CreateVolume creates a vSphere volume.
func (util *VsphereDiskUtil) CreateVolume(v *vsphereVolumeProvisioner, selectedNode *v1.Node, selectedZone []string) (volSpec *VolumeSpec, err error) {
	var fstype string
	cloud, err := getCloudProvider(v.plugin.host.GetCloudProvider())
	if err != nil {
		return nil, err
	}

	capacity := v.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	// vSphere works with KiB, but its minimum allocation unit is 1 MiB
	volSizeMiB, err := volumehelpers.RoundUpToMiBInt(capacity)
	if err != nil {
		return nil, err
	}
	volSizeKiB := volSizeMiB * 1024
	name := volumeutil.GenerateVolumeName(v.options.ClusterName, v.options.PVName, maxVolumeLength)
	volumeOptions := &vclib.VolumeOptions{
		CapacityKB: volSizeKiB,
		Tags:       *v.options.CloudTags,
		Name:       name,
	}

	volumeOptions.Zone = selectedZone
	volumeOptions.SelectedNode = selectedNode
	// Apply Parameters (case-insensitive). We leave validation of
	// the values to the cloud provider.
	for parameter, value := range v.options.Parameters {
		switch strings.ToLower(parameter) {
		case diskformat:
			volumeOptions.DiskFormat = value
		case datastore:
			volumeOptions.Datastore = value
		case volume.VolumeParameterFSType:
			fstype = value
			klog.V(4).Infof("Setting fstype as %q", fstype)
		case StoragePolicyName:
			volumeOptions.StoragePolicyName = value
			klog.V(4).Infof("Setting StoragePolicyName as %q", volumeOptions.StoragePolicyName)
		case HostFailuresToTolerateCapability, ForceProvisioningCapability,
			CacheReservationCapability, DiskStripesCapability,
			ObjectSpaceReservationCapability, IopsLimitCapability:
			capabilityData, err := validateVSANCapability(strings.ToLower(parameter), value)
			if err != nil {
				return nil, err
			}
			volumeOptions.VSANStorageProfileData += capabilityData
		default:
			return nil, fmt.Errorf("invalid option %q for volume plugin %s", parameter, v.plugin.GetPluginName())
		}
	}

	if volumeOptions.VSANStorageProfileData != "" {
		if volumeOptions.StoragePolicyName != "" {
			return nil, fmt.Errorf("cannot specify storage policy capabilities along with storage policy name. Please specify only one")
		}
		volumeOptions.VSANStorageProfileData = "(" + volumeOptions.VSANStorageProfileData + ")"
	}
	klog.V(4).Infof("VSANStorageProfileData in vsphere volume %q", volumeOptions.VSANStorageProfileData)
	// TODO: implement PVC.Selector parsing
	if v.options.PVC.Spec.Selector != nil {
		return nil, fmt.Errorf("claim.Spec.Selector is not supported for dynamic provisioning on vSphere")
	}

	vmDiskPath, err := cloud.CreateVolume(volumeOptions)
	if err != nil {
		return nil, err
	}
	labels, err := cloud.GetVolumeLabels(vmDiskPath)
	if err != nil {
		return nil, err
	}
	volSpec = &VolumeSpec{
		Path:              vmDiskPath,
		Size:              volSizeKiB,
		Fstype:            fstype,
		StoragePolicyName: volumeOptions.StoragePolicyName,
		StoragePolicyID:   volumeOptions.StoragePolicyID,
		Labels:            labels,
	}
	klog.V(2).Infof("Successfully created vsphere volume %s", name)
	return volSpec, nil
}


DeleteVolume

// DeleteVolume deletes a vSphere volume.
func (util *VsphereDiskUtil) DeleteVolume(vd *vsphereVolumeDeleter) error {
	cloud, err := getCloudProvider(vd.plugin.host.GetCloudProvider())
	if err != nil {
		return err
	}

	if err = cloud.DeleteVolume(vd.volPath); err != nil {
		klog.V(2).Infof("Error deleting vsphere volume %s: %v", vd.volPath, err)
		return err
	}
	klog.V(2).Infof("Successfully deleted vsphere volume %s", vd.volPath)
	return nil
}


getVolPathfromVolumeName

func getVolPathfromVolumeName(deviceMountPath string) string {
	// Assumption: No file or folder is named starting with '[' in datastore
	volPath := deviceMountPath[strings.LastIndex(deviceMountPath, "["):]
	// space between datastore and vmdk name in volumePath is encoded as '\040' when returned by GetMountRefs().
	// volumePath eg: "[local] xxx.vmdk" provided to attach/mount
	// replacing \040 with space to match the actual volumePath
	return strings.Replace(volPath, "\\040", " ", -1)
}


getCloudProvider

func getCloudProvider(cloud cloudprovider.Interface) (*vsphere.VSphere, error) {
	if cloud == nil {
		klog.Errorf("Cloud provider not initialized properly")
		return nil, errors.New("cloud provider not initialized properly")
	}

	vs, ok := cloud.(*vsphere.VSphere)
	if !ok || vs == nil {
		return nil, errors.New("invalid cloud provider: expected vSphere")
	}
	return vs, nil
}


validateVSANCapability

// Validate the capability requirement for the user specified policy attributes.
func validateVSANCapability(capabilityName string, capabilityValue string) (string, error) {
	var capabilityData string
	capabilityIntVal, ok := verifyCapabilityValueIsInteger(capabilityValue)
	if !ok {
		return "", fmt.Errorf("invalid value for %s. The capabilityValue: %s must be a valid integer value", capabilityName, capabilityValue)
	}
	switch strings.ToLower(capabilityName) {
	case HostFailuresToTolerateCapability:
		if capabilityIntVal >= HostFailuresToTolerateCapabilityMin && capabilityIntVal <= HostFailuresToTolerateCapabilityMax {
			capabilityData = " (\"hostFailuresToTolerate\" i" + capabilityValue + ")"
		} else {
			return "", fmt.Errorf(`invalid value for hostFailuresToTolerate.
				The default value is %d, minimum value is %d and maximum value is %d`,
				1, HostFailuresToTolerateCapabilityMin, HostFailuresToTolerateCapabilityMax)
		}
	case ForceProvisioningCapability:
		if capabilityIntVal >= ForceProvisioningCapabilityMin && capabilityIntVal <= ForceProvisioningCapabilityMax {
			capabilityData = " (\"forceProvisioning\" i" + capabilityValue + ")"
		} else {
			return "", fmt.Errorf(`invalid value for forceProvisioning.
				The value can be either %d or %d`,
				ForceProvisioningCapabilityMin, ForceProvisioningCapabilityMax)
		}
	case CacheReservationCapability:
		if capabilityIntVal >= CacheReservationCapabilityMin && capabilityIntVal <= CacheReservationCapabilityMax {
			capabilityData = " (\"cacheReservation\" i" + strconv.Itoa(capabilityIntVal*10000) + ")"
		} else {
			return "", fmt.Errorf(`invalid value for cacheReservation.
				The minimum percentage is %d and maximum percentage is %d`,
				CacheReservationCapabilityMin, CacheReservationCapabilityMax)
		}
	case DiskStripesCapability:
		if capabilityIntVal >= DiskStripesCapabilityMin && capabilityIntVal <= DiskStripesCapabilityMax {
			capabilityData = " (\"stripeWidth\" i" + capabilityValue + ")"
		} else {
			return "", fmt.Errorf(`invalid value for diskStripes.
				The minimum value is %d and maximum value is %d`,
				DiskStripesCapabilityMin, DiskStripesCapabilityMax)
		}
	case ObjectSpaceReservationCapability:
		if capabilityIntVal >= ObjectSpaceReservationCapabilityMin && capabilityIntVal <= ObjectSpaceReservationCapabilityMax {
			capabilityData = " (\"proportionalCapacity\" i" + capabilityValue + ")"
		} else {
			return "", fmt.Errorf(`invalid value for ObjectSpaceReservation.
				The minimum percentage is %d and maximum percentage is %d`,
				ObjectSpaceReservationCapabilityMin, ObjectSpaceReservationCapabilityMax)
		}
	case IopsLimitCapability:
		if capabilityIntVal >= IopsLimitCapabilityMin {
			capabilityData = " (\"iopsLimit\" i" + capabilityValue + ")"
		} else {
			return "", fmt.Errorf(`invalid value for iopsLimit.
				The value should be greater than %d`, IopsLimitCapabilityMin)
		}
	}
	return capabilityData, nil
}


verifyCapabilityValueIsInteger

// Verify if the capability value is of type integer.
func verifyCapabilityValueIsInteger(capabilityValue string) (int, bool) {
	i, err := strconv.Atoi(capabilityValue)
	if err != nil {
		return -1, false
	}
	return i, true
}


verifyDevicePath

func verifyDevicePath(path string) (string, error) {
	if pathExists, err := mount.PathExists(path); err != nil {
		return "", fmt.Errorf("error checking if path exists: %w", err)
	} else if pathExists {
		return path, nil
	}

	return "", nil
}


func verifyDevicePath(path string) (string, error) {
	return "", errors.New("unsupported")
}


func verifyDevicePath(path string) (string, error) {
	if !strings.Contains(path, diskByIDPath) {
		// If this volume has already been mounted then
		// its devicePath will have already been converted to a disk number
		klog.V(4).Infof("Found vSphere disk attached with disk number %v", path)
		return path, nil
	}
	cmd := exec.Command("powershell", "/c", "Get-Disk | Select Number, SerialNumber | ConvertTo-JSON")
	output, err := cmd.Output()
	if err != nil {
		klog.Errorf("Get-Disk failed, error: %v, output: %q", err, string(output))
		return "", err
	}

	var results []diskInfoResult
	if err = json.Unmarshal(output, &results); err != nil {
		klog.Errorf("Failed to unmarshal Get-Disk json, output: %q", string(output))
		return "", err
	}
	serialNumber := strings.TrimPrefix(path, diskByIDPath+diskSCSIPrefix)
	for _, v := range results {
		if v.SerialNumber == serialNumber {
			klog.V(4).Infof("Found vSphere disk attached with serial %v", serialNumber)
			return v.Number.String(), nil
		}
	}

	return "", fmt.Errorf("unable to find vSphere disk with serial %v", serialNumber)
}


InitService

// InitService is the entry point for running the daemon as a Windows
// service. It returns an indication of whether it is running as a service;
// and an error.
func InitService(serviceName string) error {
	h := &handler{
		tosvc:   make(chan bool),
		fromsvc: make(chan error),
	}

	var err error
	go func() {
		err = svc.Run(serviceName, h)
		h.fromsvc <- err
	}()

	// Wait for the first signal from the service handler.
	err = <-h.fromsvc
	if err != nil {
		return err
	}
	klog.Infof("Running %s as a Windows service!", serviceName)
	return nil
}


Execute

func (h *handler) Execute(_ []string, r <-chan svc.ChangeRequest, s chan<- svc.Status) (bool, uint32) {
	s <- svc.Status{State: svc.StartPending, Accepts: 0}
	// Unblock initService()
	h.fromsvc <- nil

	s <- svc.Status{State: svc.Running, Accepts: svc.AcceptStop | svc.AcceptShutdown | svc.Accepted(windows.SERVICE_ACCEPT_PARAMCHANGE)}
	klog.Infof("Service running")
Loop:
	for {
		select {
		case <-h.tosvc:
			break Loop
		case c := <-r:
			switch c.Cmd {
			case svc.Cmd(windows.SERVICE_CONTROL_PARAMCHANGE):
				s <- c.CurrentStatus
			case svc.Interrogate:
				s <- c.CurrentStatus
			case svc.Stop, svc.Shutdown:
				klog.Infof("Service stopping")
				// We need to translate this request into a signal that can be handled by the signal handler
				// handling shutdowns normally (currently apiserver/pkg/server/signal.go).
				// If we do not do this, our main threads won't be notified of the upcoming shutdown.
				// Since Windows services do not use any console, we cannot simply generate a CTRL_BREAK_EVENT
				// but need a dedicated notification mechanism.
				graceful := server.RequestShutdown()

				// Free up the control handler and let us terminate as gracefully as possible.
				// If that takes too long, the service controller will kill the remaining threads.
				// As per https://docs.microsoft.com/en-us/windows/desktop/services/service-control-handler-function
				s <- svc.Status{State: svc.StopPending}

				// If we cannot exit gracefully, we really only can exit our process, so at least the
				// service manager will think that we gracefully exited. At the time of writing this comment this is
				// needed for applications that do not use signals (e.g. kube-proxy)
				if !graceful {
					go func() {
						// Ensure the SCM was notified (The operation above (send to s) was received and communicated to the
						// service control manager - so it doesn't look like the service crashes)
						time.Sleep(1 * time.Second)
						os.Exit(0)
					}()
				}
				break Loop
			}
		}
	}

	return false, 0
}


Register

// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		return NewAlwaysAdmit(), nil
	})
}


Admit

// Admit makes an admission decision based on the request attributes
func (alwaysAdmit) Admit(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) (err error) {
	return nil
}


Validate

// Validate makes an admission decision based on the request attributes.  It is NOT allowed to mutate.
func (alwaysAdmit) Validate(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) (err error) {
	return nil
}


Handles

// Handles returns true if this admission controller can handle the given operation
// where operation can be one of CREATE, UPDATE, DELETE, or CONNECT
func (alwaysAdmit) Handles(operation admission.Operation) bool {
	return true
}


NewAlwaysAdmit

// NewAlwaysAdmit creates a new always admit admission handler
func NewAlwaysAdmit() admission.Interface {
	// DEPRECATED: AlwaysAdmit admit all admission request, it is no use.
	klog.Warningf("%s admission controller is deprecated. "+
		"Please remove this controller from your configuration files and scripts.", PluginName)
	return new(alwaysAdmit)
}


Register

// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		return NewAlwaysPullImages(), nil
	})
}


Admit

// Admit makes an admission decision based on the request attributes
func (a *AlwaysPullImages) Admit(ctx context.Context, attributes admission.Attributes, o admission.ObjectInterfaces) (err error) {
	// Ignore all calls to subresources or resources other than pods.
	if shouldIgnore(attributes) {
		return nil
	}
	pod, ok := attributes.GetObject().(*api.Pod)
	if !ok {
		return apierrors.NewBadRequest("Resource was marked with kind Pod but was unable to be converted")
	}

	pods.VisitContainersWithPath(&pod.Spec, field.NewPath("spec"), func(c *api.Container, _ *field.Path) bool {
		c.ImagePullPolicy = api.PullAlways
		return true
	})

	return nil
}


Validate

// Validate makes sure that all containers are set to always pull images
func (*AlwaysPullImages) Validate(ctx context.Context, attributes admission.Attributes, o admission.ObjectInterfaces) (err error) {
	if shouldIgnore(attributes) {
		return nil
	}

	pod, ok := attributes.GetObject().(*api.Pod)
	if !ok {
		return apierrors.NewBadRequest("Resource was marked with kind Pod but was unable to be converted")
	}

	var allErrs []error
	pods.VisitContainersWithPath(&pod.Spec, field.NewPath("spec"), func(c *api.Container, p *field.Path) bool {
		if c.ImagePullPolicy != api.PullAlways {
			allErrs = append(allErrs, admission.NewForbidden(attributes,
				field.NotSupported(p.Child("imagePullPolicy"), c.ImagePullPolicy, []string{string(api.PullAlways)}),
			))
		}
		return true
	})
	if len(allErrs) > 0 {
		return utilerrors.NewAggregate(allErrs)
	}

	return nil
}


isUpdateWithNoNewImages

// check if it's update and it doesn't change the images referenced by the pod spec
func isUpdateWithNoNewImages(attributes admission.Attributes) bool {
	if attributes.GetOperation() != admission.Update {
		return false
	}

	pod, ok := attributes.GetObject().(*api.Pod)
	if !ok {
		klog.Warningf("Resource was marked with kind Pod but pod was unable to be converted.")
		return false
	}

	oldPod, ok := attributes.GetOldObject().(*api.Pod)
	if !ok {
		klog.Warningf("Resource was marked with kind Pod but old pod was unable to be converted.")
		return false
	}

	oldImages := sets.NewString()
	pods.VisitContainersWithPath(&oldPod.Spec, field.NewPath("spec"), func(c *api.Container, _ *field.Path) bool {
		oldImages.Insert(c.Image)
		return true
	})

	hasNewImage := false
	pods.VisitContainersWithPath(&pod.Spec, field.NewPath("spec"), func(c *api.Container, _ *field.Path) bool {
		if !oldImages.Has(c.Image) {
			hasNewImage = true
		}
		return !hasNewImage
	})
	return !hasNewImage
}


shouldIgnore

func shouldIgnore(attributes admission.Attributes) bool {
	// Ignore all calls to subresources or resources other than pods.
	if len(attributes.GetSubresource()) != 0 || attributes.GetResource().GroupResource() != api.Resource("pods") {
		return true
	}

	if isUpdateWithNoNewImages(attributes) {
		return true
	}
	return false
}


NewAlwaysPullImages

// NewAlwaysPullImages creates a new always pull images admission control handler
func NewAlwaysPullImages() *AlwaysPullImages {
	return &AlwaysPullImages{
		Handler: admission.NewHandler(admission.Create, admission.Update),
	}
}


Register

// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		return NewInterPodAntiAffinity(), nil
	})
}


NewInterPodAntiAffinity

// NewInterPodAntiAffinity creates a new instance of the LimitPodHardAntiAffinityTopology admission controller
func NewInterPodAntiAffinity() *Plugin {
	return &Plugin{
		Handler: admission.NewHandler(admission.Create, admission.Update),
	}
}


Validate

// Validate will deny any pod that defines AntiAffinity topology key other than v1.LabelHostname i.e. "kubernetes.io/hostname"
// in  requiredDuringSchedulingRequiredDuringExecution and requiredDuringSchedulingIgnoredDuringExecution.
func (p *Plugin) Validate(ctx context.Context, attributes admission.Attributes, o admission.ObjectInterfaces) (err error) {
	// Ignore all calls to subresources or resources other than pods.
	if len(attributes.GetSubresource()) != 0 || attributes.GetResource().GroupResource() != api.Resource("pods") {
		return nil
	}
	pod, ok := attributes.GetObject().(*api.Pod)
	if !ok {
		return apierrors.NewBadRequest("Resource was marked with kind Pod but was unable to be converted")
	}
	affinity := pod.Spec.Affinity
	if affinity != nil && affinity.PodAntiAffinity != nil {
		var podAntiAffinityTerms []api.PodAffinityTerm
		if len(affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution) != 0 {
			podAntiAffinityTerms = affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution
		}
		// TODO: Uncomment this block when implement RequiredDuringSchedulingRequiredDuringExecution.
		//if len(affinity.PodAntiAffinity.RequiredDuringSchedulingRequiredDuringExecution) != 0 {
		//        podAntiAffinityTerms = append(podAntiAffinityTerms, affinity.PodAntiAffinity.RequiredDuringSchedulingRequiredDuringExecution...)
		//}
		for _, v := range podAntiAffinityTerms {
			if v.TopologyKey != v1.LabelHostname {
				return apierrors.NewForbidden(attributes.GetResource().GroupResource(), pod.Name, fmt.Errorf("affinity.PodAntiAffinity.RequiredDuringScheduling has TopologyKey %v but only key %v is allowed", v.TopologyKey, v1.LabelHostname))
			}
		}
	}
	return nil
}


Register

// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		return NewPlugin(), nil
	})
}


SetAuthorizer

// SetAuthorizer sets the authorizer.
func (p *Plugin) SetAuthorizer(authz authorizer.Authorizer) {
	p.authz = authz
}


ValidateInitialization

// ValidateInitialization ensures an authorizer is set.
func (p *Plugin) ValidateInitialization() error {
	if p.authz == nil {
		return fmt.Errorf("%s requires an authorizer", PluginName)
	}
	return nil
}


NewPlugin

// NewPlugin creates a new CSR approval admission plugin
func NewPlugin() *Plugin {
	return &Plugin{
		Handler: admission.NewHandler(admission.Update),
	}
}


Validate

// Validate verifies that the requesting user has permission to approve
// CertificateSigningRequests for the specified signerName.
func (p *Plugin) Validate(ctx context.Context, a admission.Attributes, _ admission.ObjectInterfaces) error {
	// Ignore all calls to anything other than 'certificatesigningrequests/approval'.
	// Ignore all operations other than UPDATE.
	if a.GetSubresource() != "approval" ||
		a.GetResource().GroupResource() != csrGroupResource {
		return nil
	}

	// We check permissions against the *old* version of the resource, in case
	// a user is attempting to update the SignerName when calling the approval
	// endpoint (which is an invalid/not allowed operation)
	csr, ok := a.GetOldObject().(*api.CertificateSigningRequest)
	if !ok {
		return admission.NewForbidden(a, fmt.Errorf("expected type CertificateSigningRequest, got: %T", a.GetOldObject()))
	}

	if !certificates.IsAuthorizedForSignerName(ctx, p.authz, a.GetUserInfo(), "approve", csr.Spec.SignerName) {
		klog.V(4).Infof("user not permitted to approve CertificateSigningRequest %q with signerName %q", csr.Name, csr.Spec.SignerName)
		return admission.NewForbidden(a, fmt.Errorf("user not permitted to approve requests with signerName %q", csr.Spec.SignerName))
	}

	return nil
}


// Validate verifies that the requesting user has permission to sign
// CertificateSigningRequests for the specified signerName.
func (p *Plugin) Validate(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) error {
	// Ignore all calls to anything other than 'certificatesigningrequests/status'.
	// Ignore all operations other than UPDATE.
	if a.GetSubresource() != "status" ||
		a.GetResource().GroupResource() != csrGroupResource {
		return nil
	}

	oldCSR, ok := a.GetOldObject().(*api.CertificateSigningRequest)
	if !ok {
		return admission.NewForbidden(a, fmt.Errorf("expected type CertificateSigningRequest, got: %T", a.GetOldObject()))
	}
	csr, ok := a.GetObject().(*api.CertificateSigningRequest)
	if !ok {
		return admission.NewForbidden(a, fmt.Errorf("expected type CertificateSigningRequest, got: %T", a.GetObject()))
	}

	// only run if the status.certificate or status.conditions field has been changed
	if reflect.DeepEqual(oldCSR.Status.Certificate, csr.Status.Certificate) && apiequality.Semantic.DeepEqual(oldCSR.Status.Conditions, csr.Status.Conditions) {
		return nil
	}

	if !certificates.IsAuthorizedForSignerName(ctx, p.authz, a.GetUserInfo(), "sign", oldCSR.Spec.SignerName) {
		klog.V(4).Infof("user not permitted to sign CertificateSigningRequest %q with signerName %q", oldCSR.Name, oldCSR.Spec.SignerName)
		return admission.NewForbidden(a, fmt.Errorf("user not permitted to sign requests with signerName %q", oldCSR.Spec.SignerName))
	}

	return nil
}


Register

// Register registers the plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		return NewPlugin(), nil
	})
}


ValidateInitialization

// ValidateInitialization always returns nil.
func (p *Plugin) ValidateInitialization() error {
	return nil
}


NewPlugin

// NewPlugin constructs a new instance of the CertificateSubjectRestrictions admission interface.
func NewPlugin() *Plugin {
	return &Plugin{
		Handler: admission.NewHandler(admission.Create),
	}
}


Validate

// Validate ensures that if the signerName on a CSR is set to
// `kubernetes.io/kube-apiserver-client`, that its organization (group)
// attribute is not set to `system:masters`.
func (p *Plugin) Validate(_ context.Context, a admission.Attributes, _ admission.ObjectInterfaces) error {
	if a.GetResource().GroupResource() != csrGroupResource || a.GetSubresource() != "" {
		return nil
	}

	csr, ok := a.GetObject().(*certificatesapi.CertificateSigningRequest)
	if !ok {
		return admission.NewForbidden(a, fmt.Errorf("expected type CertificateSigningRequest, got: %T", a.GetObject()))
	}

	if csr.Spec.SignerName != certificatesv1beta1.KubeAPIServerClientSignerName {
		return nil
	}

	csrParsed, err := certificatesapi.ParseCSR(csr.Spec.Request)
	if err != nil {
		return admission.NewForbidden(a, fmt.Errorf("failed to parse CSR: %v", err))
	}

	for _, group := range csrParsed.Subject.Organization {
		if group == "system:masters" {
			klog.V(4).Infof("CSR %s rejected by admission plugin %s for attempting to use signer %s with system:masters group",
				csr.Name, PluginName, certificatesv1beta1.KubeAPIServerClientSignerName)
			return admission.NewForbidden(a, fmt.Errorf("use of %s signer with system:masters group is not allowed",
				certificatesv1beta1.KubeAPIServerClientSignerName))
		}
	}

	return nil
}


IsAuthorizedForSignerName

// IsAuthorizedForSignerName returns true if 'info' is authorized to perform the given
// 'verb' on the synthetic 'signers' resource with the given signerName.
// If the user does not have permission to perform the 'verb' on the given signerName,
// it will also perform an authorization check against {domain portion}/*, for example
// `kubernetes.io/*`. This allows an entity to be granted permission to 'verb' on all
// signerNames with a given 'domain portion'.
func IsAuthorizedForSignerName(ctx context.Context, authz authorizer.Authorizer, info user.Info, verb, signerName string) bool {
	// First check if the user has explicit permission to 'verb' for the given signerName.
	attr := buildAttributes(info, verb, signerName)
	decision, reason, err := authz.Authorize(ctx, attr)
	switch {
	case err != nil:
		klog.V(3).Infof("cannot authorize %q %q for policy: %v,%v", verb, attr.GetName(), reason, err)
	case decision == authorizer.DecisionAllow:
		return true
	}

	// If not, check if the user has wildcard permissions to 'verb' for the domain portion of the signerName, e.g.
	// 'kubernetes.io/*'.
	attr = buildWildcardAttributes(info, verb, signerName)
	decision, reason, err = authz.Authorize(ctx, attr)
	switch {
	case err != nil:
		klog.V(3).Infof("cannot authorize %q %q for policy: %v,%v", verb, attr.GetName(), reason, err)
	case decision == authorizer.DecisionAllow:
		return true
	}

	return false
}


buildAttributes

func buildAttributes(info user.Info, verb, signerName string) authorizer.Attributes {
	return authorizer.AttributesRecord{
		User:            info,
		Verb:            verb,
		Name:            signerName,
		APIGroup:        "certificates.k8s.io",
		APIVersion:      "*",
		Resource:        "signers",
		ResourceRequest: true,
	}
}


buildWildcardAttributes

func buildWildcardAttributes(info user.Info, verb, signerName string) authorizer.Attributes {
	parts := strings.Split(signerName, "/")
	domain := parts[0]
	return buildAttributes(info, verb, domain+"/*")
}


Register

// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		return NewDefaultTolerationSeconds(), nil
	})
}


NewDefaultTolerationSeconds

// NewDefaultTolerationSeconds creates a new instance of the DefaultTolerationSeconds admission controller
func NewDefaultTolerationSeconds() *Plugin {
	return &Plugin{
		Handler: admission.NewHandler(admission.Create, admission.Update),
	}
}


Admit

// Admit makes an admission decision based on the request attributes
func (p *Plugin) Admit(ctx context.Context, attributes admission.Attributes, o admission.ObjectInterfaces) (err error) {
	if attributes.GetResource().GroupResource() != api.Resource("pods") {
		return nil
	}

	if len(attributes.GetSubresource()) > 0 {
		// only run the checks below on pods proper and not subresources
		return nil
	}

	pod, ok := attributes.GetObject().(*api.Pod)
	if !ok {
		return errors.NewBadRequest(fmt.Sprintf("expected *api.Pod but got %T", attributes.GetObject()))
	}

	tolerations := pod.Spec.Tolerations

	toleratesNodeNotReady := false
	toleratesNodeUnreachable := false
	for _, toleration := range tolerations {
		if (toleration.Key == v1.TaintNodeNotReady || len(toleration.Key) == 0) &&
			(toleration.Effect == api.TaintEffectNoExecute || len(toleration.Effect) == 0) {
			toleratesNodeNotReady = true
		}

		if (toleration.Key == v1.TaintNodeUnreachable || len(toleration.Key) == 0) &&
			(toleration.Effect == api.TaintEffectNoExecute || len(toleration.Effect) == 0) {
			toleratesNodeUnreachable = true
		}
	}

	if !toleratesNodeNotReady {
		pod.Spec.Tolerations = append(pod.Spec.Tolerations, notReadyToleration)
	}

	if !toleratesNodeUnreachable {
		pod.Spec.Tolerations = append(pod.Spec.Tolerations, unreachableToleration)
	}

	return nil
}


Register

// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		return NewAlwaysDeny(), nil
	})
}


Admit

// Admit makes an admission decision based on the request attributes.
func (alwaysDeny) Admit(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) (err error) {
	return admission.NewForbidden(a, errors.New("admission control is denying all modifications"))
}


Validate

// Validate makes an admission decision based on the request attributes.  It is NOT allowed to mutate.
func (alwaysDeny) Validate(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) (err error) {
	return admission.NewForbidden(a, errors.New("admission control is denying all modifications"))
}


Handles

// Handles returns true if this admission controller can handle the given operation
// where operation can be one of CREATE, UPDATE, DELETE, or CONNECT
func (alwaysDeny) Handles(operation admission.Operation) bool {
	return true
}


NewAlwaysDeny

// NewAlwaysDeny creates an always deny admission handler
func NewAlwaysDeny() admission.Interface {
	// DEPRECATED: AlwaysDeny denys all admission request, it is no use.
	klog.Warningf("%s admission controller is deprecated. "+
		"Please remove this controller from your configuration files and scripts.", PluginName)
	return new(alwaysDeny)
}


Register

// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName,
		func(config io.Reader) (admission.Interface, error) {
			// load the configuration provided (if any)
			configuration, err := LoadConfiguration(config)
			if err != nil {
				return nil, err
			}
			// validate the configuration (if any)
			if configuration != nil {
				if errs := validation.ValidateConfiguration(configuration); len(errs) != 0 {
					return nil, errs.ToAggregate()
				}
			}
			return newEventRateLimit(configuration, clock.RealClock{})
		})
}


newEventRateLimit

// newEventRateLimit configures an admission controller that can enforce event rate limits
func newEventRateLimit(config *eventratelimitapi.Configuration, clock flowcontrol.Clock) (*Plugin, error) {
	limitEnforcers := make([]*limitEnforcer, 0, len(config.Limits))
	for _, limitConfig := range config.Limits {
		enforcer, err := newLimitEnforcer(limitConfig, clock)
		if err != nil {
			return nil, err
		}
		limitEnforcers = append(limitEnforcers, enforcer)
	}

	eventRateLimitAdmission := &Plugin{
		Handler:        admission.NewHandler(admission.Create, admission.Update),
		limitEnforcers: limitEnforcers,
	}

	return eventRateLimitAdmission, nil
}


Validate

// Validate makes admission decisions while enforcing event rate limits
func (a *Plugin) Validate(ctx context.Context, attr admission.Attributes, o admission.ObjectInterfaces) (err error) {
	// ignore all operations that do not correspond to an Event kind
	if attr.GetKind().GroupKind() != api.Kind("Event") {
		return nil
	}

	// ignore all requests that specify dry-run
	// because they don't correspond to any calls to etcd,
	// they should not be affected by the ratelimit
	if attr.IsDryRun() {
		return nil
	}

	var errors []error
	// give each limit enforcer a chance to reject the event
	for _, enforcer := range a.limitEnforcers {
		if err := enforcer.accept(attr); err != nil {
			errors = append(errors, err)
		}
	}

	if aggregatedErr := utilerrors.NewAggregate(errors); aggregatedErr != nil {
		return apierrors.NewTooManyRequestsError(aggregatedErr.Error())
	}

	return nil
}


Install

// Install registers the API group and adds types to a scheme
func Install(scheme *runtime.Scheme) {
	utilruntime.Must(internalapi.AddToScheme(scheme))
	utilruntime.Must(versionedapi.AddToScheme(scheme))
	utilruntime.Must(scheme.SetVersionPriority(versionedapi.SchemeGroupVersion))
}


Kind

// Kind takes an unqualified kind and returns a Group qualified GroupKind
func Kind(kind string) schema.GroupKind {
	return SchemeGroupVersion.WithKind(kind).GroupKind()
}


Resource

// Resource takes an unqualified resource and returns a Group qualified GroupResource
func Resource(resource string) schema.GroupResource {
	return SchemeGroupVersion.WithResource(resource).GroupResource()
}


addKnownTypes

func addKnownTypes(scheme *runtime.Scheme) error {
	// TODO this will get cleaned up with the scheme types are fixed
	scheme.AddKnownTypes(SchemeGroupVersion,
		&Configuration{},
	)
	return nil
}


addDefaultingFuncs

func addDefaultingFuncs(scheme *kruntime.Scheme) error {
	return RegisterDefaults(scheme)
}


func SetDefaults_Configuration(obj *Configuration) {}


init

func init() {
	// We only register manually written functions here. The registration of the
	// generated functions takes place in the generated files. The separation
	// makes the code compile even when the generated files are missing.
	localSchemeBuilder.Register(addKnownTypes, addDefaultingFuncs)
}


addKnownTypes

func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&Configuration{},
	)
	return nil
}


ValidateConfiguration

// ValidateConfiguration validates the configuration.
func ValidateConfiguration(config *eventratelimitapi.Configuration) field.ErrorList {
	allErrs := field.ErrorList{}
	limitsPath := field.NewPath("limits")
	if len(config.Limits) == 0 {
		allErrs = append(allErrs, field.Invalid(limitsPath, config.Limits, "must not be empty"))
	}
	for i, limit := range config.Limits {
		idxPath := limitsPath.Index(i)
		if !limitTypes[limit.Type] {
			allowedValues := make([]string, len(limitTypes))
			i := 0
			for limitType := range limitTypes {
				allowedValues[i] = string(limitType)
				i++
			}
			allErrs = append(allErrs, field.NotSupported(idxPath.Child("type"), limit.Type, allowedValues))
		}
		if limit.Burst <= 0 {
			allErrs = append(allErrs, field.Invalid(idxPath.Child("burst"), limit.Burst, "must be positive"))
		}
		if limit.QPS <= 0 {
			allErrs = append(allErrs, field.Invalid(idxPath.Child("qps"), limit.QPS, "must be positive"))
		}
		if limit.Type != eventratelimitapi.ServerLimitType {
			if limit.CacheSize < 0 {
				allErrs = append(allErrs, field.Invalid(idxPath.Child("cacheSize"), limit.CacheSize, "must not be negative"))
			}
		}
	}
	return allErrs
}


get

func (c *singleCache) get(key interface{}) flowcontrol.RateLimiter {
	return c.rateLimiter
}


func (c *lruCache) get(key interface{}) flowcontrol.RateLimiter {
	value, found := c.cache.Get(key)
	if !found {
		rateLimter := c.rateLimiterFactory()
		c.cache.Add(key, rateLimter)
		return rateLimter
	}
	return value.(flowcontrol.RateLimiter)
}


init

func init() {
	install.Install(scheme)
}


LoadConfiguration

// LoadConfiguration loads the provided configuration.
func LoadConfiguration(config io.Reader) (*eventratelimitapi.Configuration, error) {
	// if no config is provided, return a default configuration
	if config == nil {
		externalConfig := &eventratelimitv1alpha1.Configuration{}
		scheme.Default(externalConfig)
		internalConfig := &eventratelimitapi.Configuration{}
		if err := scheme.Convert(externalConfig, internalConfig, nil); err != nil {
			return nil, err
		}
		return internalConfig, nil
	}
	// we have a config so parse it.
	data, err := ioutil.ReadAll(config)
	if err != nil {
		return nil, err
	}
	decoder := codecs.UniversalDecoder()
	decodedObj, err := runtime.Decode(decoder, data)
	if err != nil {
		return nil, err
	}
	resourceQuotaConfiguration, ok := decodedObj.(*eventratelimitapi.Configuration)
	if !ok {
		return nil, fmt.Errorf("unexpected type: %T", decodedObj)
	}
	return resourceQuotaConfiguration, nil
}


newLimitEnforcer

func newLimitEnforcer(config eventratelimitapi.Limit, clock flowcontrol.Clock) (*limitEnforcer, error) {
	rateLimiterFactory := func() flowcontrol.RateLimiter {
		return flowcontrol.NewTokenBucketRateLimiterWithClock(float32(config.QPS), int(config.Burst), clock)
	}

	if config.Type == eventratelimitapi.ServerLimitType {
		return &limitEnforcer{
			limitType: config.Type,
			cache: &singleCache{
				rateLimiter: rateLimiterFactory(),
			},
			keyFunc: getServerKey,
		}, nil
	}

	cacheSize := int(config.CacheSize)
	if cacheSize == 0 {
		cacheSize = defaultCacheSize
	}
	underlyingCache := lru.New(cacheSize)
	cache := &lruCache{
		rateLimiterFactory: rateLimiterFactory,
		cache:              underlyingCache,
	}

	var keyFunc func(admission.Attributes) string
	switch t := config.Type; t {
	case eventratelimitapi.NamespaceLimitType:
		keyFunc = getNamespaceKey
	case eventratelimitapi.UserLimitType:
		keyFunc = getUserKey
	case eventratelimitapi.SourceAndObjectLimitType:
		keyFunc = getSourceAndObjectKey
	default:
		return nil, fmt.Errorf("unknown event rate limit type: %v", t)
	}

	return &limitEnforcer{
		limitType: config.Type,
		cache:     cache,
		keyFunc:   keyFunc,
	}, nil
}


accept

func (enforcer *limitEnforcer) accept(attr admission.Attributes) error {
	key := enforcer.keyFunc(attr)
	rateLimiter := enforcer.cache.get(key)

	// ensure we have available rate
	allow := rateLimiter.TryAccept()

	if !allow {
		return fmt.Errorf("limit reached on type %v for key %v", enforcer.limitType, key)
	}

	return nil
}


getServerKey

func getServerKey(attr admission.Attributes) string {
	return ""
}


getNamespaceKey

// getNamespaceKey returns a cache key that is based on the namespace of the event request
func getNamespaceKey(attr admission.Attributes) string {
	return attr.GetNamespace()
}


getUserKey

// getUserKey returns a cache key that is based on the user of the event request
func getUserKey(attr admission.Attributes) string {
	userInfo := attr.GetUserInfo()
	if userInfo == nil {
		return ""
	}
	return userInfo.GetName()
}


getSourceAndObjectKey

// getSourceAndObjectKey returns a cache key that is based on the source+object of the event
func getSourceAndObjectKey(attr admission.Attributes) string {
	object := attr.GetObject()
	if object == nil {
		return ""
	}
	event, ok := object.(*api.Event)
	if !ok {
		return ""
	}
	return strings.Join([]string{
		event.Source.Component,
		event.Source.Host,
		event.InvolvedObject.Kind,
		event.InvolvedObject.Namespace,
		event.InvolvedObject.Name,
		string(event.InvolvedObject.UID),
		event.InvolvedObject.APIVersion,
	}, "")
}


Register

// Register is called by the apiserver to register the plugin factory.
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		return newExtendedResourceToleration(), nil
	})
}


newExtendedResourceToleration

// newExtendedResourceToleration creates a new instance of the ExtendedResourceToleration admission controller.
func newExtendedResourceToleration() *plugin {
	return &plugin{
		Handler: admission.NewHandler(admission.Create, admission.Update),
	}
}


Admit

// Admit updates the toleration of a pod based on the resources requested by it.
// If an extended resource of name "example.com/device" is requested, it adds
// a toleration with key "example.com/device", operator "Exists" and effect "NoSchedule".
// The rationale for this is described in:
// https://github.com/kubernetes/kubernetes/issues/55080
func (p *plugin) Admit(ctx context.Context, attributes admission.Attributes, o admission.ObjectInterfaces) error {
	// Ignore all calls to subresources or resources other than pods.
	if len(attributes.GetSubresource()) != 0 || attributes.GetResource().GroupResource() != core.Resource("pods") {
		return nil
	}

	pod, ok := attributes.GetObject().(*core.Pod)
	if !ok {
		return errors.NewBadRequest(fmt.Sprintf("expected *core.Pod but got %T", attributes.GetObject()))
	}

	resources := sets.String{}
	for _, container := range pod.Spec.Containers {
		for resourceName := range container.Resources.Requests {
			if helper.IsExtendedResourceName(resourceName) {
				resources.Insert(string(resourceName))
			}
		}
	}
	for _, container := range pod.Spec.InitContainers {
		for resourceName := range container.Resources.Requests {
			if helper.IsExtendedResourceName(resourceName) {
				resources.Insert(string(resourceName))
			}
		}
	}

	// Doing .List() so that we get a stable sorted list.
	// This allows us to test adding tolerations for multiple extended resources.
	for _, resource := range resources.List() {
		helper.AddOrUpdateTolerationInPod(pod, &core.Toleration{
			Key:      resource,
			Operator: core.TolerationOpExists,
			Effect:   core.TaintEffectNoSchedule,
		})
	}

	return nil
}


Register

// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		// the pods/status endpoint is ignored by this plugin since old kubelets
		// corrupt them.  the pod status strategy ensures status updates cannot mutate
		// ownerRef.
		whiteList := []whiteListItem{
			{
				groupResource: schema.GroupResource{Resource: "pods"},
				subresource:   "status",
			},
		}
		return &gcPermissionsEnforcement{
			Handler:   admission.NewHandler(admission.Create, admission.Update),
			whiteList: whiteList,
		}, nil
	})
}


isWhiteListed

// isWhiteListed returns true if the specified item is in the whitelist.
func (a *gcPermissionsEnforcement) isWhiteListed(groupResource schema.GroupResource, subresource string) bool {
	for _, item := range a.whiteList {
		if item.groupResource == groupResource && item.subresource == subresource {
			return true
		}
	}
	return false
}


Validate

func (a *gcPermissionsEnforcement) Validate(ctx context.Context, attributes admission.Attributes, o admission.ObjectInterfaces) (err error) {
	// // if the request is in the whitelist, we skip mutation checks for this resource.
	if a.isWhiteListed(attributes.GetResource().GroupResource(), attributes.GetSubresource()) {
		return nil
	}

	// if we aren't changing owner references, then the edit is always allowed
	if !isChangingOwnerReference(attributes.GetObject(), attributes.GetOldObject()) {
		return nil
	}

	// if you are creating a thing, you should always be allowed to set an owner ref since you logically had the power
	// to never create it.  We still need to check block owner deletion below, because the power to delete does not
	// imply the power to prevent deletion on other resources.
	if attributes.GetOperation() != admission.Create {
		deleteAttributes := authorizer.AttributesRecord{
			User:            attributes.GetUserInfo(),
			Verb:            "delete",
			Namespace:       attributes.GetNamespace(),
			APIGroup:        attributes.GetResource().Group,
			APIVersion:      attributes.GetResource().Version,
			Resource:        attributes.GetResource().Resource,
			Subresource:     attributes.GetSubresource(),
			Name:            attributes.GetName(),
			ResourceRequest: true,
			Path:            "",
		}
		decision, reason, err := a.authorizer.Authorize(ctx, deleteAttributes)
		if decision != authorizer.DecisionAllow {
			return admission.NewForbidden(attributes, fmt.Errorf("cannot set an ownerRef on a resource you can't delete: %v, %v", reason, err))
		}
	}

	// Further check if the user is setting ownerReference.blockOwnerDeletion to
	// true. If so, only allows the change if the user has delete permission of
	// the _OWNER_
	newBlockingRefs := newBlockingOwnerDeletionRefs(attributes.GetObject(), attributes.GetOldObject())
	if len(newBlockingRefs) == 0 {
		return nil
	}

	// There can be a case where a restMapper tries to hit discovery endpoints and times out if the network is inaccessible.
	// This can prevent creating the pod to run the network to be able to do discovery and it appears as a timeout, not a rejection.
	// Because the timeout is wrapper on admission/request, we can run a single check to see if the user can finalize any
	// possible resource.
	if decision, _, _ := a.authorizer.Authorize(ctx, finalizeAnythingRecord(attributes.GetUserInfo())); decision == authorizer.DecisionAllow {
		return nil
	}

	for _, ref := range newBlockingRefs {
		records, err := a.ownerRefToDeleteAttributeRecords(ref, attributes)
		if err != nil {
			return admission.NewForbidden(attributes, fmt.Errorf("cannot set blockOwnerDeletion in this case because cannot find RESTMapping for APIVersion %s Kind %s: %v", ref.APIVersion, ref.Kind, err))
		}
		// Multiple records are returned if ref.Kind could map to multiple
		// resources. User needs to have delete permission on all the
		// matched Resources.
		for _, record := range records {
			decision, reason, err := a.authorizer.Authorize(ctx, record)
			if decision != authorizer.DecisionAllow {
				return admission.NewForbidden(attributes, fmt.Errorf("cannot set blockOwnerDeletion if an ownerReference refers to a resource you can't set finalizers on: %v, %v", reason, err))
			}
		}
	}

	return nil

}


isChangingOwnerReference

func isChangingOwnerReference(newObj, oldObj runtime.Object) bool {
	newMeta, err := meta.Accessor(newObj)
	if err != nil {
		// if we don't have objectmeta, we don't have the object reference
		return false
	}

	if oldObj == nil {
		return len(newMeta.GetOwnerReferences()) > 0
	}
	oldMeta, err := meta.Accessor(oldObj)
	if err != nil {
		// if we don't have objectmeta, we don't have the object reference
		return false
	}

	// compare the old and new.  If they aren't the same, then we're trying to change an ownerRef
	oldOwners := oldMeta.GetOwnerReferences()
	newOwners := newMeta.GetOwnerReferences()
	if len(oldOwners) != len(newOwners) {
		return true
	}
	for i := range oldOwners {
		if !apiequality.Semantic.DeepEqual(oldOwners[i], newOwners[i]) {
			return true
		}
	}

	return false
}


finalizeAnythingRecord

func finalizeAnythingRecord(userInfo user.Info) authorizer.AttributesRecord {
	return authorizer.AttributesRecord{
		User:            userInfo,
		Verb:            "update",
		APIGroup:        "*",
		APIVersion:      "*",
		Resource:        "*",
		Subresource:     "finalizers",
		Name:            "*",
		ResourceRequest: true,
		Path:            "",
	}
}


ownerRefToDeleteAttributeRecords

// Translates ref to a DeleteAttribute deleting the object referred by the ref.
// OwnerReference only records the object kind, which might map to multiple
// resources, so multiple DeleteAttribute might be returned.
func (a *gcPermissionsEnforcement) ownerRefToDeleteAttributeRecords(ref metav1.OwnerReference, attributes admission.Attributes) ([]authorizer.AttributesRecord, error) {
	var ret []authorizer.AttributesRecord
	groupVersion, err := schema.ParseGroupVersion(ref.APIVersion)
	if err != nil {
		return ret, err
	}
	mappings, err := a.restMapper.RESTMappings(schema.GroupKind{Group: groupVersion.Group, Kind: ref.Kind}, groupVersion.Version)
	if err != nil {
		return ret, err
	}
	for _, mapping := range mappings {
		ar := authorizer.AttributesRecord{
			User:            attributes.GetUserInfo(),
			Verb:            "update",
			APIGroup:        mapping.Resource.Group,
			APIVersion:      mapping.Resource.Version,
			Resource:        mapping.Resource.Resource,
			Subresource:     "finalizers",
			Name:            ref.Name,
			ResourceRequest: true,
			Path:            "",
		}
		if mapping.Scope.Name() == meta.RESTScopeNameNamespace {
			// if the owner is namespaced, it must be in the same namespace as the dependent is.
			ar.Namespace = attributes.GetNamespace()
		}
		ret = append(ret, ar)
	}
	return ret, nil
}


blockingOwnerRefs

// only keeps the blocking refs
func blockingOwnerRefs(refs []metav1.OwnerReference) []metav1.OwnerReference {
	var ret []metav1.OwnerReference
	for _, ref := range refs {
		if ref.BlockOwnerDeletion != nil && *ref.BlockOwnerDeletion == true {
			ret = append(ret, ref)
		}
	}
	return ret
}


indexByUID

func indexByUID(refs []metav1.OwnerReference) map[types.UID]metav1.OwnerReference {
	ret := make(map[types.UID]metav1.OwnerReference)
	for _, ref := range refs {
		ret[ref.UID] = ref
	}
	return ret
}


newBlockingOwnerDeletionRefs

// Returns new blocking ownerReferences, and references whose blockOwnerDeletion
// field is changed from nil or false to true.
func newBlockingOwnerDeletionRefs(newObj, oldObj runtime.Object) []metav1.OwnerReference {
	newMeta, err := meta.Accessor(newObj)
	if err != nil {
		// if we don't have objectmeta, we don't have the object reference
		return nil
	}
	newRefs := newMeta.GetOwnerReferences()
	blockingNewRefs := blockingOwnerRefs(newRefs)
	if len(blockingNewRefs) == 0 {
		return nil
	}

	if oldObj == nil {
		return blockingNewRefs
	}
	oldMeta, err := meta.Accessor(oldObj)
	if err != nil {
		// if we don't have objectmeta, treat it as if all the ownerReference are newly created
		return blockingNewRefs
	}

	var ret []metav1.OwnerReference
	indexedOldRefs := indexByUID(oldMeta.GetOwnerReferences())
	for _, ref := range blockingNewRefs {
		oldRef, ok := indexedOldRefs[ref.UID]
		if !ok {
			// if ref is newly added, and it's blocking, then returns it.
			ret = append(ret, ref)
			continue
		}
		wasNotBlocking := oldRef.BlockOwnerDeletion == nil || *oldRef.BlockOwnerDeletion == false
		if wasNotBlocking {
			ret = append(ret, ref)
		}
	}
	return ret
}


SetAuthorizer

func (a *gcPermissionsEnforcement) SetAuthorizer(authorizer authorizer.Authorizer) {
	a.authorizer = authorizer
}


SetRESTMapper

func (a *gcPermissionsEnforcement) SetRESTMapper(restMapper meta.RESTMapper) {
	a.restMapper = restMapper
}


ValidateInitialization

func (a *gcPermissionsEnforcement) ValidateInitialization() error {
	if a.authorizer == nil {
		return fmt.Errorf("missing authorizer")
	}
	if a.restMapper == nil {
		return fmt.Errorf("missing restMapper")
	}
	return nil
}


Register

// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		newImagePolicyWebhook, err := NewImagePolicyWebhook(config)
		if err != nil {
			return nil, err
		}
		return newImagePolicyWebhook, nil
	})
}


statusTTL

func (a *Plugin) statusTTL(status v1alpha1.ImageReviewStatus) time.Duration {
	if status.Allowed {
		return a.allowTTL
	}
	return a.denyTTL
}


filterAnnotations

// Filter out annotations that don't match *.image-policy.k8s.io/*
func (a *Plugin) filterAnnotations(allAnnotations map[string]string) map[string]string {
	annotations := make(map[string]string)
	for k, v := range allAnnotations {
		if strings.Contains(k, ".image-policy.k8s.io/") {
			annotations[k] = v
		}
	}
	return annotations
}


webhookError

// Function to call on webhook failure; behavior determined by defaultAllow flag
func (a *Plugin) webhookError(pod *api.Pod, attributes admission.Attributes, err error) error {
	if err != nil {
		klog.V(2).Infof("error contacting webhook backend: %s", err)
		if a.defaultAllow {
			attributes.AddAnnotation(AuditKeyPrefix+ImagePolicyFailedOpenKeySuffix, "true")
			// TODO(wteiken): Remove the annotation code for the 1.13 release
			annotations := pod.GetAnnotations()
			if annotations == nil {
				annotations = make(map[string]string)
			}
			annotations[api.ImagePolicyFailedOpenKey] = "true"
			pod.ObjectMeta.SetAnnotations(annotations)

			klog.V(2).Infof("resource allowed in spite of webhook backend failure")
			return nil
		}
		klog.V(2).Infof("resource not allowed due to webhook backend failure ")
		return admission.NewForbidden(attributes, err)
	}
	return nil
}


Validate

// Validate makes an admission decision based on the request attributes
func (a *Plugin) Validate(ctx context.Context, attributes admission.Attributes, o admission.ObjectInterfaces) (err error) {
	// Ignore all calls to subresources other than ephemeralcontainers or calls to resources other than pods.
	subresource := attributes.GetSubresource()
	if (subresource != "" && subresource != ephemeralcontainers) || attributes.GetResource().GroupResource() != api.Resource("pods") {
		return nil
	}

	pod, ok := attributes.GetObject().(*api.Pod)
	if !ok {
		return apierrors.NewBadRequest("Resource was marked with kind Pod but was unable to be converted")
	}

	// Build list of ImageReviewContainerSpec
	var imageReviewContainerSpecs []v1alpha1.ImageReviewContainerSpec
	if subresource == "" {
		containers := make([]api.Container, 0, len(pod.Spec.Containers)+len(pod.Spec.InitContainers))
		containers = append(containers, pod.Spec.Containers...)
		containers = append(containers, pod.Spec.InitContainers...)
		for _, c := range containers {
			imageReviewContainerSpecs = append(imageReviewContainerSpecs, v1alpha1.ImageReviewContainerSpec{
				Image: c.Image,
			})
		}
	} else if subresource == ephemeralcontainers {
		for _, c := range pod.Spec.EphemeralContainers {
			imageReviewContainerSpecs = append(imageReviewContainerSpecs, v1alpha1.ImageReviewContainerSpec{
				Image: c.Image,
			})
		}
	}
	imageReview := v1alpha1.ImageReview{
		Spec: v1alpha1.ImageReviewSpec{
			Containers:  imageReviewContainerSpecs,
			Annotations: a.filterAnnotations(pod.Annotations),
			Namespace:   attributes.GetNamespace(),
		},
	}
	if err := a.admitPod(ctx, pod, attributes, &imageReview); err != nil {
		return admission.NewForbidden(attributes, err)
	}
	return nil
}


admitPod

func (a *Plugin) admitPod(ctx context.Context, pod *api.Pod, attributes admission.Attributes, review *v1alpha1.ImageReview) error {
	cacheKey, err := json.Marshal(review.Spec)
	if err != nil {
		return err
	}
	if entry, ok := a.responseCache.Get(string(cacheKey)); ok {
		review.Status = entry.(v1alpha1.ImageReviewStatus)
	} else {
		result := a.webhook.WithExponentialBackoff(ctx, func() rest.Result {
			return a.webhook.RestClient.Post().Body(review).Do(ctx)
		})

		if err := result.Error(); err != nil {
			return a.webhookError(pod, attributes, err)
		}
		var statusCode int
		if result.StatusCode(&statusCode); statusCode < 200 || statusCode >= 300 {
			return a.webhookError(pod, attributes, fmt.Errorf("Error contacting webhook: %d", statusCode))
		}

		if err := result.Into(review); err != nil {
			return a.webhookError(pod, attributes, err)
		}

		a.responseCache.Add(string(cacheKey), review.Status, a.statusTTL(review.Status))
	}

	for k, v := range review.Status.AuditAnnotations {
		if err := attributes.AddAnnotation(AuditKeyPrefix+k, v); err != nil {
			klog.Warningf("failed to set admission audit annotation %s to %s: %v", AuditKeyPrefix+k, v, err)
		}
	}
	if !review.Status.Allowed {
		if len(review.Status.Reason) > 0 {
			return fmt.Errorf("image policy webhook backend denied one or more images: %s", review.Status.Reason)
		}
		return errors.New("one or more images rejected by webhook backend")
	}
	return nil
}


NewImagePolicyWebhook

// NewImagePolicyWebhook a new ImagePolicyWebhook plugin from the provided config file.
// The config file is specified by --admission-control-config-file and has the
// following format for a webhook:
//
//	{
//	  "imagePolicy": {
//	     "kubeConfigFile": "path/to/kubeconfig/for/backend",
//	     "allowTTL": 30,           # time in s to cache approval
//	     "denyTTL": 30,            # time in s to cache denial
//	     "retryBackoff": 500,      # time in ms to wait between retries
//	     "defaultAllow": true      # determines behavior if the webhook backend fails
//	  }
//	}
//
// The config file may be json or yaml.
//
// The kubeconfig property refers to another file in the kubeconfig format which
// specifies how to connect to the webhook backend.
//
// The kubeconfig's cluster field is used to refer to the remote service, user refers to the returned authorizer.
//
//	# clusters refers to the remote service.
//	clusters:
//	- name: name-of-remote-imagepolicy-service
//	  cluster:
//	    certificate-authority: /path/to/ca.pem      # CA for verifying the remote service.
//	    server: https://images.example.com/policy # URL of remote service to query. Must use 'https'.
//
//	# users refers to the API server's webhook configuration.
//	users:
//	- name: name-of-api-server
//	  user:
//	    client-certificate: /path/to/cert.pem # cert for the webhook plugin to use
//	    client-key: /path/to/key.pem          # key matching the cert
//
// For additional HTTP configuration, refer to the kubeconfig documentation
// http://kubernetes.io/v1.1/docs/user-guide/kubeconfig-file.html.
func NewImagePolicyWebhook(configFile io.Reader) (*Plugin, error) {
	if configFile == nil {
		return nil, fmt.Errorf("no config specified")
	}

	// TODO: move this to a versioned configuration file format
	var config AdmissionConfig
	d := yaml.NewYAMLOrJSONDecoder(configFile, 4096)
	err := d.Decode(&config)
	if err != nil {
		return nil, err
	}

	whConfig := config.ImagePolicyWebhook
	if err := normalizeWebhookConfig(&whConfig); err != nil {
		return nil, err
	}

	clientConfig, err := webhook.LoadKubeconfig(whConfig.KubeConfigFile, nil)
	if err != nil {
		return nil, err
	}
	retryBackoff := webhook.DefaultRetryBackoffWithInitialDelay(whConfig.RetryBackoff)
	gw, err := webhook.NewGenericWebhook(legacyscheme.Scheme, legacyscheme.Codecs, clientConfig, groupVersions, retryBackoff)
	if err != nil {
		return nil, err
	}
	return &Plugin{
		Handler:       admission.NewHandler(admission.Create, admission.Update),
		webhook:       gw,
		responseCache: cache.NewLRUExpireCache(1024),
		allowTTL:      whConfig.AllowTTL,
		denyTTL:       whConfig.DenyTTL,
		defaultAllow:  whConfig.DefaultAllow,
	}, nil
}


normalizeWebhookConfig

func normalizeWebhookConfig(config *imagePolicyWebhookConfig) (err error) {
	config.RetryBackoff, err = normalizeConfigDuration("backoff", time.Millisecond, config.RetryBackoff, minRetryBackoff, maxRetryBackoff, defaultRetryBackoff)
	if err != nil {
		return err
	}
	config.AllowTTL, err = normalizeConfigDuration("allow cache", time.Second, config.AllowTTL, minAllowTTL, maxAllowTTL, defaultAllowTTL)
	if err != nil {
		return err
	}
	config.DenyTTL, err = normalizeConfigDuration("deny cache", time.Second, config.DenyTTL, minDenyTTL, maxDenyTTL, defaultDenyTTL)
	return err
}


normalizeConfigDuration

func normalizeConfigDuration(name string, scale, value, min, max, defaultValue time.Duration) (time.Duration, error) {
	// disable with -1 sentinel
	if value == disableTTL {
		klog.V(2).Infof("image policy webhook %s disabled", name)
		return time.Duration(0), nil
	}

	// use default with 0 sentinel
	if value == useDefault {
		klog.V(2).Infof("image policy webhook %s using default value", name)
		return defaultValue, nil
	}

	// convert to s; unmarshalling gives ns
	value *= scale

	// check value is within range
	if value < min || value > max {
		return value, fmt.Errorf("valid value is between %v and %v, got %v", min, max, value)
	}
	return value, nil
}


Register

// Register registers a plugin
func Register(plugins *admission.Plugins) {
	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
		return NewLimitRanger(&DefaultLimitRangerActions{})
	})
}


SetExternalKubeInformerFactory

// SetExternalKubeInformerFactory registers an informer factory into the LimitRanger
func (l *LimitRanger) SetExternalKubeInformerFactory(f informers.SharedInformerFactory) {
	limitRangeInformer := f.Core().V1().LimitRanges()
	l.SetReadyFunc(limitRangeInformer.Informer().HasSynced)
	l.lister = limitRangeInformer.Lister()
}


SetExternalKubeClientSet

// SetExternalKubeClientSet registers the client into LimitRanger
func (l *LimitRanger) SetExternalKubeClientSet(client kubernetes.Interface) {
	l.client = client
}


ValidateInitialization

// ValidateInitialization verifies the LimitRanger object has been properly initialized
func (l *LimitRanger) ValidateInitialization() error {
	if l.lister == nil {
		return fmt.Errorf("missing limitRange lister")
	}
	if l.client == nil {
		return fmt.Errorf("missing client")
	}
	return nil
}


Admit

// Admit admits resources into cluster that do not violate any defined LimitRange in the namespace
func (l *LimitRanger) Admit(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) (err error) {
	return l.runLimitFunc(a, l.actions.MutateLimit)
}


Validate

// Validate admits resources into cluster that do not violate any defined LimitRange in the namespace
func (l *LimitRanger) Validate(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) (err error) {
	return l.runLimitFunc(a, l.actions.ValidateLimit)
}


runLimitFunc

func (l *LimitRanger) runLimitFunc(a admission.Attributes, limitFn func(limitRange *corev1.LimitRange, kind string, obj runtime.Object) error) (err error) {
	if !l.actions.SupportsAttributes(a) {
		return nil
	}

	// ignore all objects marked for deletion
	oldObj := a.GetOldObject()
	if oldObj != nil {
		oldAccessor, err := meta.Accessor(oldObj)
		if err != nil {
			return admission.NewForbidden(a, err)
		}
		if oldAccessor.GetDeletionTimestamp() != nil {
			return nil
		}
	}

	items, err := l.GetLimitRanges(a)
	if err != nil {
		return err
	}

	// ensure it meets each prescribed min/max
	for i := range items {
		limitRange := items[i]

		if !l.actions.SupportsLimit(limitRange) {
			continue
		}

		err = limitFn(limitRange, a.GetResource().Resource, a.GetObject())
		if err != nil {
			return admission.NewForbidden(a, err)
		}
	}
	return nil
}


GetLimitRanges

// GetLimitRanges returns a LimitRange object with the items held in
// the indexer if available, or do alive lookup of the value.
func (l *LimitRanger) GetLimitRanges(a admission.Attributes) ([]*corev1.LimitRange, error) {
	items, err := l.lister.LimitRanges(a.GetNamespace()).List(labels.Everything())
	if err != nil {
		return nil, admission.NewForbidden(a, fmt.Errorf("unable to %s %v at this time because there was an error enforcing limit ranges", a.GetOperation(), a.GetResource()))
	}

	// if there are no items held in our indexer, check our live-lookup LRU, if that misses, do the live lookup to prime it.
	if len(items) == 0 {
		lruItemObj, ok := l.liveLookupCache.Get(a.GetNamespace())
		if !ok || lruItemObj.(liveLookupEntry).expiry.Before(time.Now()) {
			// Fixed: #22422
			// use singleflight to alleviate simultaneous calls to
			lruItemObj, err, _ = l.group.Do(a.GetNamespace(), func() (interface{}, error) {
				liveList, err := l.client.CoreV1().LimitRanges(a.GetNamespace()).List(context.TODO(), metav1.ListOptions{})
				if err != nil {
					return nil, admission.NewForbidden(a, err)
				}
				newEntry := liveLookupEntry{expiry: time.Now().Add(l.liveTTL)}
				for i := range liveList.Items {
					newEntry.items = append(newEntry.items, &liveList.Items[i])
				}
				l.liveLookupCache.Add(a.GetNamespace(), newEntry)
				return newEntry, nil
			})
			if err != nil {
				return nil, err
			}
		}
		lruEntry := lruItemObj.(liveLookupEntry)

		for i := range lruEntry.items {
			items = append(items, lruEntry.items[i])
		}

	}

	return items, nil
}


NewLimitRanger

// NewLimitRanger returns an object that enforces limits based on the supplied limit function
func NewLimitRanger(actions LimitRangerActions) (*LimitRanger, error) {
	liveLookupCache := lru.New(10000)

	if actions == nil {
		actions = &DefaultLimitRangerActions{}
	}

	return &LimitRanger{
		Handler:         admission.NewHandler(admission.Create, admission.Update),
		actions:         actions,
		liveLookupCache: liveLookupCache,
		liveTTL:         time.Duration(30 * time.Second),
	}, nil
}


defaultContainerResourceRequirements

// defaultContainerResourceRequirements returns the default requirements for a container
// the requirement.Limits are taken from the LimitRange defaults (if specified)
// the requirement.Requests are taken from the LimitRange default request (if specified)
func defaultContainerResourceRequirements(limitRange *corev1.LimitRange) api.ResourceRequirements {
	requirements := api.ResourceRequirements{}
	requirements.Requests = api.ResourceList{}
	requirements.Limits = api.ResourceList{}

	for i := range limitRange.Spec.Limits {
		limit := limitRange.Spec.Limits[i]
		if limit.Type == corev1.LimitTypeContainer {
			for k, v := range limit.DefaultRequest {
				requirements.Requests[api.ResourceName(k)] = v.DeepCopy()
			}
			for k, v := range limit.Default {
				requirements.Limits[api.ResourceName(k)] = v.DeepCopy()
			}
		}
	}
	return requirements
}


mergeContainerResources

// mergeContainerResources handles defaulting all of the resources on a container.
func mergeContainerResources(container *api.Container, defaultRequirements *api.ResourceRequirements, annotationPrefix string, annotations []string) []string {
	setRequests := []string{}
	setLimits := []string{}
	if container.Resources.Limits == nil {
		container.Resources.Limits = api.ResourceList{}
	}
	if container.Resources.Requests == nil {
		container.Resources.Requests = api.ResourceList{}
	}
	for k, v := range defaultRequirements.Limits {
		_, found := container.Resources.Limits[k]
		if !found {
			container.Resources.Limits[k] = v.DeepCopy()
			setLimits = append(setLimits, string(k))
		}
	}
	for k, v := range defaultRequirements.Requests {
		_, found := container.Resources.Requests[k]
		if !found {
			container.Resources.Requests[k] = v.DeepCopy()
			setRequests = append(setRequests, string(k))
		}
	}
	if len(setRequests) > 0 {
		sort.Strings(setRequests)
		a := strings.Join(setRequests, ", ") + fmt.Sprintf(" request for %s %s", annotationPrefix, container.Name)
		annotations = append(annotations, a)
	}
	if len(setLimits) > 0 {
		sort.Strings(setLimits)
		a := strings.Join(setLimits, ", ") + fmt.Sprintf(" limit for %s %s", annotationPrefix, container.Name)
		annotations = append(annotations, a)
	}
	return annotations
}


mergePodResourceRequirements

// mergePodResourceRequirements merges enumerated requirements with default requirements
// it annotates the pod with information about what requirements were modified
func mergePodResourceRequirements(pod *api.Pod, defaultRequirements *api.ResourceRequirements) {
	annotations := []string{}

	for i := range pod.Spec.Containers {
		annotations = mergeContainerResources(&pod.Spec.Containers[i], defaultRequirements, "container", annotations)
	}

	for i := range pod.Spec.InitContainers {
		annotations = mergeContainerResources(&pod.Spec.InitContainers[i], defaultRequirements, "init container", annotations)
	}

	if len(annotations) > 0 {
		if pod.ObjectMeta.Annotations == nil {
			pod.ObjectMeta.Annotations = make(map[string]string)
		}
		val := "LimitRanger plugin set: " + strings.Join(annotations, "; ")
		pod.ObjectMeta.Annotations[limitRangerAnnotation] = val
	}
}


requestLimitEnforcedValues

// requestLimitEnforcedValues returns the specified values at a common precision to support comparability
func requestLimitEnforcedValues(requestQuantity, limitQuantity, enforcedQuantity resource.Quantity) (request, limit, enforced int64) {
	request = requestQuantity.Value()
	limit = limitQuantity.Value()
	enforced = enforcedQuantity.Value()
	// do a more precise comparison if possible (if the value won't overflow)
	if request <= resource.MaxMilliValue && limit <= resource.MaxMilliValue && enforced <= resource.MaxMilliValue {
		request = requestQuantity.MilliValue()
		limit = limitQuantity.MilliValue()
		enforced = enforcedQuantity.MilliValue()
	}
	return
}


minConstraint

// minConstraint enforces the min constraint over the specified resource
func minConstraint(limitType string, resourceName string, enforced resource.Quantity, request api.ResourceList, limit api.ResourceList) error {
	req, reqExists := request[api.ResourceName(resourceName)]
	lim, limExists := limit[api.ResourceName(resourceName)]
	observedReqValue, observedLimValue, enforcedValue := requestLimitEnforcedValues(req, lim, enforced)

	if !reqExists {
		return fmt.Errorf("minimum %s usage per %s is %s.  No request is specified", resourceName, limitType, enforced.String())
	}
	if observedReqValue < enforcedValue {
		return fmt.Errorf("minimum %s usage per %s is %s, but request is %s", resourceName, limitType, enforced.String(), req.String())
	}
	if limExists && (observedLimValue < enforcedValue) {
		return fmt.Errorf("minimum %s usage per %s is %s, but limit is %s", resourceName, limitType, enforced.String(), lim.String())
	}
	return nil
}


maxRequestConstraint

// maxRequestConstraint enforces the max constraint over the specified resource
// use when specify LimitType resource doesn't recognize limit values
func maxRequestConstraint(limitType string, resourceName string, enforced resource.Quantity, request api.ResourceList) error {
	req, reqExists := request[api.ResourceName(resourceName)]
	observedReqValue, _, enforcedValue := requestLimitEnforcedValues(req, resource.Quantity{}, enforced)

	if !reqExists {
		return fmt.Errorf("maximum %s usage per %s is %s.  No request is specified", resourceName, limitType, enforced.String())
	}
	if observedReqValue > enforcedValue {
		return fmt.Errorf("maximum %s usage per %s is %s, but request is %s", resourceName, limitType, enforced.String(), req.String())
	}
	return nil
}


maxConstraint

// maxConstraint enforces the max constraint over the specified resource
func maxConstraint(limitType string, resourceName string, enforced resource.Quantity, request api.ResourceList, limit api.ResourceList) error {
	req, reqExists := request[api.ResourceName(resourceName)]
	lim, limExists := limit[api.ResourceName(resourceName)]
	observedReqValue, observedLimValue, enforcedValue := requestLimitEnforcedValues(req, lim, enforced)

	if !limExists {
		return fmt.Errorf("maximum %s usage per %s is %s.  No limit is specified", resourceName, limitType, enforced.String())
	}
	if observedLimValue > enforcedValue {
		return fmt.Errorf("maximum %s usage per %s is %s, but limit is %s", resourceName, limitType, enforced.String(), lim.String())
	}
	if reqExists && (observedReqValue > enforcedValue) {
		return fmt.Errorf("maximum %s usage per %s is %s, but request is %s", resourceName, limitType, enforced.String(), req.String())
	}
	return nil
}