func createISCSIDisk(spec *volume.Spec, podUID types.UID, plugin *iscsiPlugin, manager diskManager, secret map[string]string) (*iscsiDisk, error) {
	tp, portals, iqn, lunStr, err := getISCSITargetInfo(spec)
	if err != nil {
		return nil, err
	}

	lun := strconv.Itoa(int(lunStr))
	portal := portalMounter(tp)
	var bkportal []string
	bkportal = append(bkportal, portal)
	for _, p := range portals {
		bkportal = append(bkportal, portalMounter(string(p)))
	}

	iface, initiatorNamePtr, err := getISCSIInitiatorInfo(spec)
	if err != nil {
		return nil, err
	}

	var initiatorName string
	if initiatorNamePtr != nil {
		initiatorName = *initiatorNamePtr
	}
	chapDiscovery, err := getISCSIDiscoveryCHAPInfo(spec)
	if err != nil {
		return nil, err
	}
	chapSession, err := getISCSISessionCHAPInfo(spec)
	if err != nil {
		return nil, err
	}

	initIface := iface
	if initiatorName != "" {
		iface = bkportal[0] + ":" + spec.Name()
	}

	return &iscsiDisk{
		podUID:        podUID,
		VolName:       spec.Name(),
		Portals:       bkportal,
		Iqn:           iqn,
		Lun:           lun,
		InitIface:     initIface,
		Iface:         iface,
		chapDiscovery: chapDiscovery,
		chapSession:   chapSession,
		secret:        secret,
		InitiatorName: initiatorName,
		manager:       manager,
		plugin:        plugin}, nil
}


func createSecretMap(spec *volume.Spec, plugin *iscsiPlugin, namespace string) (map[string]string, error) {
	var secret map[string]string
	chapDiscover, err := getISCSIDiscoveryCHAPInfo(spec)
	if err != nil {
		return nil, err
	}
	chapSession, err := getISCSISessionCHAPInfo(spec)
	if err != nil {
		return nil, err
	}
	if chapDiscover || chapSession {
		secretName, secretNamespace, err := getISCSISecretNameAndNamespace(spec, namespace)
		if err != nil {
			return nil, err
		}

		if len(secretName) > 0 && len(secretNamespace) > 0 {
			// if secret is provideded, retrieve it
			kubeClient := plugin.host.GetKubeClient()
			if kubeClient == nil {
				return nil, fmt.Errorf("cannot get kube client")
			}
			secretObj, err := kubeClient.CoreV1().Secrets(secretNamespace).Get(context.TODO(), secretName, metav1.GetOptions{})
			if err != nil {
				err = fmt.Errorf("couldn't get secret %v/%v error: %w", secretNamespace, secretName, err)
				return nil, err
			}
			secret = make(map[string]string)
			for name, data := range secretObj.Data {
				klog.V(4).Infof("retrieving CHAP secret name: %s", name)
				secret[name] = string(data)
			}
		}
	}
	return secret, err
}


func createPersistentVolumeFromISCSIPVSource(volumeName string, iscsi v1.ISCSIPersistentVolumeSource) *v1.PersistentVolume {
	block := v1.PersistentVolumeBlock
	return &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: volumeName,
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeSource: v1.PersistentVolumeSource{
				ISCSI: &iscsi,
			},
			VolumeMode: &block,
		},
	}
}


func getVolumeSpecFromGlobalMapPath(volumeName, globalMapPath string) (*volume.Spec, error) {
	// Retrieve volume spec information from globalMapPath
	// globalMapPath example:
	// plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}
	// plugins/kubernetes.io/iscsi/volumeDevices/iface-default/192.168.0.10:3260-iqn.2017-05.com.example:test-lun-0

	// device: 192.168.0.10:3260-iqn.2017-05.com.example:test-lun-0
	device, _, err := extractDeviceAndPrefix(globalMapPath)
	if err != nil {
		return nil, err
	}
	bkpPortal, iqn, err := extractPortalAndIqn(device)
	if err != nil {
		return nil, err
	}
	arr := strings.Split(device, "-lun-")
	if len(arr) < 2 {
		return nil, fmt.Errorf("failed to retrieve lun from globalMapPath: %v", globalMapPath)
	}
	lun, err := strconv.Atoi(arr[1])
	if err != nil {
		return nil, err
	}
	iface, found := extractIface(globalMapPath)
	if !found {
		return nil, fmt.Errorf("failed to retrieve iface from globalMapPath: %v", globalMapPath)
	}
	iscsiPV := createPersistentVolumeFromISCSIPVSource(volumeName,
		v1.ISCSIPersistentVolumeSource{
			TargetPortal:   bkpPortal,
			IQN:            iqn,
			Lun:            int32(lun),
			ISCSIInterface: iface,
		},
	)
	klog.V(5).Infof("ConstructBlockVolumeSpec: TargetPortal: %v, IQN: %v, Lun: %v, ISCSIInterface: %v",
		iscsiPV.Spec.PersistentVolumeSource.ISCSI.TargetPortal,
		iscsiPV.Spec.PersistentVolumeSource.ISCSI.IQN,
		iscsiPV.Spec.PersistentVolumeSource.ISCSI.Lun,
		iscsiPV.Spec.PersistentVolumeSource.ISCSI.ISCSIInterface,
	)
	return volume.NewSpecFromPersistentVolume(iscsiPV, false), nil
}


func updateISCSIDiscoverydb(b iscsiDiskMounter, tp string) error {
	if !b.chapDiscovery {
		return nil
	}
	out, err := execWithLog(b, "iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "update", "-n", "discovery.sendtargets.auth.authmethod", "-v", "CHAP")
	if err != nil {
		return fmt.Errorf("iscsi: failed to update discoverydb with CHAP, output: %v", out)
	}

	for _, k := range chapSt {
		v := b.secret[k]
		if len(v) > 0 {
			// explicitly not using execWithLog so secrets are not logged
			out, err := b.exec.Command("iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "update", "-n", k, "-v", v).CombinedOutput()
			if err != nil {
				return fmt.Errorf("iscsi: failed to update discoverydb key %q error: %v", k, string(out))
			}
		}
	}
	return nil
}


func updateISCSINode(b iscsiDiskMounter, tp string) error {
	// setting node.session.scan to manual to handle https://github.com/kubernetes/kubernetes/issues/90982
	out, err := execWithLog(b, "iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "-o", "update", "-n", "node.session.scan", "-v", "manual")
	if err != nil {
		// don't fail if iscsiadm fails or the version does not support node.session.scan - log a warning to highlight the potential exposure
		klog.Warningf("iscsi: failed to update node with node.session.scan=manual, possible exposure to issue 90982: %v", out)
	}

	if !b.chapSession {
		return nil
	}

	out, err = execWithLog(b, "iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "-o", "update", "-n", "node.session.auth.authmethod", "-v", "CHAP")
	if err != nil {
		return fmt.Errorf("iscsi: failed to update node with CHAP, output: %v", out)
	}

	for _, k := range chapSess {
		v := b.secret[k]
		if len(v) > 0 {
			// explicitly not using execWithLog so secrets are not logged
			out, err := b.exec.Command("iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "-o", "update", "-n", k, "-v", v).CombinedOutput()
			if err != nil {
				return fmt.Errorf("iscsi: failed to update node session key %q error: %v", k, string(out))
			}
		}
	}
	return nil
}


func waitForPathToExist(devicePath *string, maxRetries int, deviceTransport string) bool {
	// This makes unit testing a lot easier
	return waitForPathToExistInternal(devicePath, maxRetries, deviceTransport, os.Stat, filepath.Glob)
}


func waitForPathToExistInternal(devicePath *string, maxRetries int, deviceTransport string, osStat StatFunc, filepathGlob GlobFunc) bool {
	if devicePath == nil {
		return false
	}

	for i := 0; i < maxRetries; i++ {
		var err error
		if deviceTransport == "tcp" {
			_, err = osStat(*devicePath)
		} else {
			fpath, _ := filepathGlob(*devicePath)
			if fpath == nil {
				err = os.ErrNotExist
			} else {
				// There might be a case that fpath contains multiple device paths if
				// multiple PCI devices connect to same iscsi target. We handle this
				// case at subsequent logic. Pick up only first path here.
				*devicePath = fpath[0]
			}
		}
		if err == nil {
			return true
		}
		if !os.IsNotExist(err) {
			return false
		}
		if i == maxRetries-1 {
			break
		}
		time.Sleep(time.Second)
	}
	return false
}


// make a directory like /var/lib/kubelet/plugins/kubernetes.io/iscsi/iface_name/portal-some_iqn-lun-lun_id
func makePDNameInternal(host volume.VolumeHost, portal string, iqn string, lun string, iface string) string {
	return filepath.Join(host.GetPluginDir(iscsiPluginName), "iface-"+iface, portal+"-"+iqn+"-lun-"+lun)
}


// make a directory like /var/lib/kubelet/plugins/kubernetes.io/iscsi/volumeDevices/iface_name/portal-some_iqn-lun-lun_id
func makeVDPDNameInternal(host volume.VolumeHost, portal string, iqn string, lun string, iface string) string {
	return filepath.Join(host.GetVolumeDevicePluginDir(iscsiPluginName), "iface-"+iface, portal+"-"+iqn+"-lun-"+lun)
}


// MakeGlobalPDName returns path of global plugin dir
func (util *ISCSIUtil) MakeGlobalPDName(iscsi iscsiDisk) string {
	return makePDNameInternal(iscsi.plugin.host, iscsi.Portals[0], iscsi.Iqn, iscsi.Lun, iscsi.Iface)
}


// MakeGlobalVDPDName returns path of global volume device plugin dir
func (util *ISCSIUtil) MakeGlobalVDPDName(iscsi iscsiDisk) string {
	return makeVDPDNameInternal(iscsi.plugin.host, iscsi.Portals[0], iscsi.Iqn, iscsi.Lun, iscsi.Iface)
}


// persistISCSIFile saves iSCSI volume configuration for DetachDisk
// into given directory.
func (util *ISCSIUtil) persistISCSIFile(conf iscsiDisk, mnt string) error {
	file := filepath.Join(mnt, "iscsi.json")
	fp, err := os.Create(file)
	if err != nil {
		return fmt.Errorf("iscsi: create %s err %s", file, err)
	}
	defer fp.Close()
	encoder := json.NewEncoder(fp)
	if err = encoder.Encode(conf); err != nil {
		return fmt.Errorf("iscsi: encode err: %v", err)
	}
	return nil
}


func (util *ISCSIUtil) loadISCSI(conf *iscsiDisk, mnt string) error {
	file := filepath.Join(mnt, "iscsi.json")
	fp, err := os.Open(file)
	if err != nil {
		return fmt.Errorf("iscsi: open %s err %s", file, err)
	}
	defer fp.Close()
	decoder := json.NewDecoder(fp)
	if err = decoder.Decode(conf); err != nil {
		return fmt.Errorf("iscsi: decode err: %v", err)
	}
	return nil
}


// scanOneLun scans a single LUN on one SCSI bus
// Use this to avoid scanning the whole SCSI bus for all of the LUNs, which
// would result in the kernel on this node discovering LUNs that it shouldn't
// know about. Extraneous LUNs cause problems because they may get deleted
// without us getting notified, since we were never supposed to know about
// them. When LUNs are deleted without proper cleanup in the kernel, I/O errors
// and timeouts result, which can noticeably degrade performance of future
// operations.
func scanOneLun(hostNumber int, lunNumber int) error {
	filename := fmt.Sprintf("/sys/class/scsi_host/host%d/scan", hostNumber)
	fd, err := os.OpenFile(filename, os.O_WRONLY, 0)
	if err != nil {
		return err
	}
	defer fd.Close()

	// Channel/Target are always 0 for iSCSI
	scanCmd := fmt.Sprintf("0 0 %d", lunNumber)
	if written, err := fd.WriteString(scanCmd); err != nil {
		return err
	} else if 0 == written {
		return fmt.Errorf("no data written to file: %s", filename)
	}

	klog.V(3).Infof("Scanned SCSI host %d LUN %d", hostNumber, lunNumber)
	return nil
}


func waitForMultiPathToExist(devicePaths []string, maxRetries int, deviceUtil volumeutil.DeviceUtil) string {
	if 0 == len(devicePaths) {
		return ""
	}

	for i := 0; i < maxRetries; i++ {
		for _, path := range devicePaths {
			// There shouldn't be any empty device paths. However adding this check
			// for safer side to avoid the possibility of an empty entry.
			if path == "" {
				continue
			}
			// check if the dev is using mpio and if so mount it via the dm-XX device
			if mappedDevicePath := deviceUtil.FindMultipathDeviceForDevice(path); mappedDevicePath != "" {
				return mappedDevicePath
			}
		}
		if i == maxRetries-1 {
			break
		}
		time.Sleep(time.Second)
	}
	return ""
}


// AttachDisk returns devicePath of volume if attach succeeded otherwise returns error
func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) (string, error) {
	var devicePath string
	devicePaths := map[string]string{}
	var iscsiTransport string
	var lastErr error

	out, err := execWithLog(b, "iscsiadm", "-m", "iface", "-I", b.InitIface, "-o", "show")
	if err != nil {
		klog.Errorf("iscsi: could not read iface %s error: %s", b.InitIface, out)
		return "", err
	}

	iscsiTransport = extractTransportname(out)

	bkpPortal := b.Portals

	// If the initiator name was set, the iface isn't created yet,
	// so create it and copy parameters from the pre-configured one
	if b.InitiatorName != "" {
		if err = cloneIface(b); err != nil {
			klog.Errorf("iscsi: failed to clone iface: %s error: %v", b.InitIface, err)
			return "", err
		}
	}

	// Lock the target while we login to avoid races between 2 volumes that share the same
	// target both logging in or one logging out while another logs in.
	b.plugin.targetLocks.LockKey(b.Iqn)
	defer b.plugin.targetLocks.UnlockKey(b.Iqn)

	// Build a map of SCSI hosts for each target portal. We will need this to
	// issue the bus rescans.
	portalHostMap, err := b.deviceUtil.GetISCSIPortalHostMapForTarget(b.Iqn)
	if err != nil {
		return "", err
	}
	klog.V(4).Infof("AttachDisk portal->host map for %s is %v", b.Iqn, portalHostMap)

	for i := 1; i <= maxAttachAttempts; i++ {
		for _, tp := range bkpPortal {
			if _, found := devicePaths[tp]; found {
				klog.V(4).Infof("Device for portal %q already known", tp)
				continue
			}

			hostNumber, loggedIn := portalHostMap[tp]
			if !loggedIn {
				klog.V(4).Infof("Could not get SCSI host number for portal %s, will attempt login", tp)

				// build discoverydb and discover iscsi target
				execWithLog(b, "iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "new")

				// update discoverydb with CHAP secret
				err = updateISCSIDiscoverydb(b, tp)
				if err != nil {
					lastErr = fmt.Errorf("iscsi: failed to update discoverydb to portal %s error: %v", tp, err)
					continue
				}

				out, err = execWithLog(b, "iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "--discover")
				if err != nil {
					// delete discoverydb record
					execWithLog(b, "iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "delete")
					lastErr = fmt.Errorf("iscsi: failed to sendtargets to portal %s output: %s, err %v", tp, out, err)
					continue
				}

				err = updateISCSINode(b, tp)
				if err != nil {
					// failure to update node db is rare. But deleting record will likely impact those who already start using it.
					lastErr = fmt.Errorf("iscsi: failed to update iscsi node to portal %s error: %v", tp, err)
					continue
				}

				// login to iscsi target
				out, err = execWithLog(b, "iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "--login")
				if err != nil {
					// delete the node record from database
					execWithLog(b, "iscsiadm", "-m", "node", "-p", tp, "-I", b.Iface, "-T", b.Iqn, "-o", "delete")
					lastErr = fmt.Errorf("iscsi: failed to attach disk: Error: %s (%v)", out, err)
					continue
				}

				// in case of node failure/restart, explicitly set to manual login so it doesn't hang on boot
				_, err = execWithLog(b, "iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-o", "update", "-n", "node.startup", "-v", "manual")
				if err != nil {
					// don't fail if we can't set startup mode, but log warning so there is a clue
					klog.Warningf("Warning: Failed to set iSCSI login mode to manual. Error: %v", err)
				}

				// Rebuild the host map after logging in
				portalHostMap, err := b.deviceUtil.GetISCSIPortalHostMapForTarget(b.Iqn)
				if err != nil {
					return "", err
				}
				klog.V(6).Infof("AttachDisk portal->host map for %s is %v", b.Iqn, portalHostMap)

				hostNumber, loggedIn = portalHostMap[tp]
				if !loggedIn {
					klog.Warningf("Could not get SCSI host number for portal %s after logging in", tp)
					continue
				}
			}

			klog.V(5).Infof("AttachDisk: scanning SCSI host %d LUN %s", hostNumber, b.Lun)
			lunNumber, err := strconv.Atoi(b.Lun)
			if err != nil {
				return "", fmt.Errorf("AttachDisk: lun is not a number: %s\nError: %v", b.Lun, err)
			}

			// Scan the iSCSI bus for the LUN
			err = scanOneLun(hostNumber, lunNumber)
			if err != nil {
				return "", err
			}

			if iscsiTransport == "" {
				klog.Errorf("iscsi: could not find transport name in iface %s", b.Iface)
				return "", fmt.Errorf("could not parse iface file for %s", b.Iface)
			}

			addr := tp
			if strings.HasPrefix(tp, "[") {
				// Delete [] from IP address, links in /dev/disk/by-path do not have it.
				addr = strings.NewReplacer("[", "", "]", "").Replace(tp)
			}
			if iscsiTransport == "tcp" {
				devicePath = strings.Join([]string{"/dev/disk/by-path/ip", addr, "iscsi", b.Iqn, "lun", b.Lun}, "-")
			} else {
				devicePath = strings.Join([]string{"/dev/disk/by-path/pci", "*", "ip", addr, "iscsi", b.Iqn, "lun", b.Lun}, "-")
			}

			if exist := waitForPathToExist(&devicePath, deviceDiscoveryTimeout, iscsiTransport); !exist {
				msg := fmt.Sprintf("Timed out waiting for device at path %s after %ds", devicePath, deviceDiscoveryTimeout)
				klog.Error(msg)
				// update last error
				lastErr = errors.New(msg)
				continue
			} else {
				devicePaths[tp] = devicePath
			}
		}
		klog.V(4).Infof("iscsi: tried all devices for %q %d times, %d paths found", b.Iqn, i, len(devicePaths))
		if len(devicePaths) == 0 {
			// No path attached, report error and stop trying. kubelet will try again in a short while
			// delete cloned iface
			execWithLog(b, "iscsiadm", "-m", "iface", "-I", b.Iface, "-o", "delete")
			klog.Errorf("iscsi: failed to get any path for iscsi disk, last err seen:\n%v", lastErr)
			return "", fmt.Errorf("failed to get any path for iscsi disk, last err seen:\n%v", lastErr)
		}
		if len(devicePaths) == len(bkpPortal) {
			// We have all paths
			klog.V(4).Infof("iscsi: all devices for %q found", b.Iqn)
			break
		}
		if len(devicePaths) >= minMultipathCount && i >= minAttachAttempts {
			// We have at least two paths for multipath and we tried the other paths long enough
			klog.V(4).Infof("%d devices found for %q", len(devicePaths), b.Iqn)
			break
		}
	}

	if lastErr != nil {
		klog.Errorf("iscsi: last error occurred during iscsi init:\n%v", lastErr)
	}

	devicePathList := []string{}
	for _, path := range devicePaths {
		devicePathList = append(devicePathList, path)
	}
	// Try to find a multipath device for the volume
	if len(bkpPortal) > 1 {
		// Multipath volume was requested. Wait up to multipathDeviceTimeout seconds for the multipath device to appear.
		devicePath = waitForMultiPathToExist(devicePathList, multipathDeviceTimeout, b.deviceUtil)
	} else {
		// For PVs with 1 portal, just try one time to find the multipath device. This
		// avoids a long pause when the multipath device will never get created, and
		// matches legacy behavior.
		devicePath = waitForMultiPathToExist(devicePathList, 1, b.deviceUtil)
	}

	// When no multipath device is found, just use the first (and presumably only) device
	if devicePath == "" {
		devicePath = devicePathList[0]
	}

	klog.V(5).Infof("iscsi: AttachDisk devicePath: %s", devicePath)

	if err = util.persistISCSI(b); err != nil {
		// Return uncertain error so kubelet calls Unmount / Unmap when the pod
		// is deleted.
		return "", types.NewUncertainProgressError(err.Error())
	}
	return devicePath, nil
}


// persistISCSI saves iSCSI volume configuration for DetachDisk into global
// mount / map directory.
func (util *ISCSIUtil) persistISCSI(b iscsiDiskMounter) error {
	klog.V(5).Infof("iscsi: AttachDisk volumeMode: %s", b.volumeMode)
	var globalPDPath string
	if b.volumeMode == v1.PersistentVolumeBlock {
		globalPDPath = b.manager.MakeGlobalVDPDName(*b.iscsiDisk)
	} else {
		globalPDPath = b.manager.MakeGlobalPDName(*b.iscsiDisk)
	}

	if err := os.MkdirAll(globalPDPath, 0750); err != nil {
		klog.Errorf("iscsi: failed to mkdir %s, error", globalPDPath)
		return err
	}

	if b.volumeMode == v1.PersistentVolumeFilesystem {
		notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
		if err != nil {
			return err
		}
		if !notMnt {
			// The volume is already mounted, therefore the previous WaitForAttach must have
			// persisted the volume metadata. In addition, the metadata is actually *inside*
			// globalPDPath and we can't write it here, because it was shadowed by the volume
			// mount.
			klog.V(4).Infof("Skipping persistISCSI, the volume is already mounted at %s", globalPDPath)
			return nil
		}
	}

	// Persist iscsi disk config to json file for DetachDisk path
	return util.persistISCSIFile(*(b.iscsiDisk), globalPDPath)
}


// Delete 1 block device of the form "sd*"
func deleteDevice(deviceName string) error {
	filename := fmt.Sprintf("/sys/block/%s/device/delete", deviceName)
	fd, err := os.OpenFile(filename, os.O_WRONLY, 0)
	if err != nil {
		// The file was not present, so just return without error
		return nil
	}
	defer fd.Close()

	if written, err := fd.WriteString("1"); err != nil {
		return err
	} else if 0 == written {
		return fmt.Errorf("no data written to file: %s", filename)
	}
	klog.V(4).Infof("Deleted block device: %s", deviceName)
	return nil
}


// deleteDevices tries to remove all the block devices and multipath map devices
// associated with a given iscsi device
func deleteDevices(c iscsiDiskUnmounter) error {
	lunNumber, err := strconv.Atoi(c.iscsiDisk.Lun)
	if err != nil {
		klog.Errorf("iscsi delete devices: lun is not a number: %s\nError: %v", c.iscsiDisk.Lun, err)
		return err
	}
	// Enumerate the devices so we can delete them
	deviceNames, err := c.deviceUtil.FindDevicesForISCSILun(c.iscsiDisk.Iqn, lunNumber)
	if err != nil {
		klog.Errorf("iscsi delete devices: could not get devices associated with LUN %d on target %s\nError: %v",
			lunNumber, c.iscsiDisk.Iqn, err)
		return err
	}
	// Find the multipath device path(s)
	mpathDevices := make(map[string]bool)
	for _, deviceName := range deviceNames {
		path := "/dev/" + deviceName
		// check if the dev is using mpio and if so mount it via the dm-XX device
		if mappedDevicePath := c.deviceUtil.FindMultipathDeviceForDevice(path); mappedDevicePath != "" {
			mpathDevices[mappedDevicePath] = true
		}
	}
	// Flush any multipath device maps
	for mpathDevice := range mpathDevices {
		_, err = c.exec.Command("multipath", "-f", mpathDevice).CombinedOutput()
		if err != nil {
			klog.Warningf("Warning: Failed to flush multipath device map: %s\nError: %v", mpathDevice, err)
			// Fall through -- keep deleting the block devices
		}
		klog.V(4).Infof("Flushed multipath device: %s", mpathDevice)
	}
	for _, deviceName := range deviceNames {
		err = deleteDevice(deviceName)
		if err != nil {
			klog.Warningf("Warning: Failed to delete block device: %s\nError: %v", deviceName, err)
			// Fall through -- keep deleting other block devices
		}
	}
	return nil
}


// DetachDisk unmounts and detaches a volume from node
func (util *ISCSIUtil) DetachDisk(c iscsiDiskUnmounter, mntPath string) error {
	if pathExists, pathErr := mount.PathExists(mntPath); pathErr != nil {
		return fmt.Errorf("error checking if path exists: %w", pathErr)
	} else if !pathExists {
		klog.Warningf("Warning: Unmount skipped because path does not exist: %v", mntPath)
		return nil
	}

	notMnt, err := c.mounter.IsLikelyNotMountPoint(mntPath)
	if err != nil {
		return err
	}
	if !notMnt {
		if err := c.mounter.Unmount(mntPath); err != nil {
			klog.Errorf("iscsi detach disk: failed to unmount: %s\nError: %v", mntPath, err)
			return err
		}
	}

	// if device is no longer used, see if need to logout the target
	device, _, err := extractDeviceAndPrefix(mntPath)
	if err != nil {
		return err
	}

	var bkpPortal []string
	var volName, iqn, iface, initiatorName string
	found := true

	// load iscsi disk config from json file
	if err := util.loadISCSI(c.iscsiDisk, mntPath); err == nil {
		bkpPortal, iqn, iface, volName = c.iscsiDisk.Portals, c.iscsiDisk.Iqn, c.iscsiDisk.Iface, c.iscsiDisk.VolName
		initiatorName = c.iscsiDisk.InitiatorName
	} else {
		// If the iscsi disk config is not found, fall back to the original behavior.
		// This portal/iqn/iface is no longer referenced, log out.
		// Extract the portal and iqn from device path.
		bkpPortal = make([]string, 1)
		bkpPortal[0], iqn, err = extractPortalAndIqn(device)
		if err != nil {
			return err
		}
		// Extract the iface from the mountPath and use it to log out. If the iface
		// is not found, maintain the previous behavior to facilitate kubelet upgrade.
		// Logout may fail as no session may exist for the portal/IQN on the specified interface.
		iface, found = extractIface(mntPath)
	}

	// Delete all the scsi devices and any multipath devices after unmounting
	if err = deleteDevices(c); err != nil {
		klog.Warningf("iscsi detach disk: failed to delete devices\nError: %v", err)
		// Fall through -- even if deleting fails, a logout may fix problems
	}

	// Lock the target while we determine if we can safely log out or not
	c.plugin.targetLocks.LockKey(iqn)
	defer c.plugin.targetLocks.UnlockKey(iqn)

	portals := removeDuplicate(bkpPortal)
	if len(portals) == 0 {
		return fmt.Errorf("iscsi detach disk: failed to detach iscsi disk. Couldn't get connected portals from configurations")
	}

	// If device is no longer used, see if need to logout the target
	if isSessionBusy(c.iscsiDisk.plugin.host, portals[0], iqn) {
		return nil
	}

	err = util.detachISCSIDisk(c.exec, portals, iqn, iface, volName, initiatorName, found)
	if err != nil {
		return fmt.Errorf("failed to finish detachISCSIDisk, err: %v", err)
	}
	return nil
}


// DetachBlockISCSIDisk removes loopback device for a volume and detaches a volume from node
func (util *ISCSIUtil) DetachBlockISCSIDisk(c iscsiDiskUnmapper, mapPath string) error {
	if pathExists, pathErr := mount.PathExists(mapPath); pathErr != nil {
		return fmt.Errorf("error checking if path exists: %w", pathErr)
	} else if !pathExists {
		klog.Warningf("Warning: Unmap skipped because path does not exist: %v", mapPath)
		return nil
	}
	// If we arrive here, device is no longer used, see if need to logout the target
	// device: 192.168.0.10:3260-iqn.2017-05.com.example:test-lun-0
	device, _, err := extractDeviceAndPrefix(mapPath)
	if err != nil {
		return err
	}
	var bkpPortal []string
	var volName, iqn, lun, iface, initiatorName string
	found := true
	// load iscsi disk config from json file
	if err := util.loadISCSI(c.iscsiDisk, mapPath); err == nil {
		bkpPortal, iqn, lun, iface, volName = c.iscsiDisk.Portals, c.iscsiDisk.Iqn, c.iscsiDisk.Lun, c.iscsiDisk.Iface, c.iscsiDisk.VolName
		initiatorName = c.iscsiDisk.InitiatorName
	} else {
		// If the iscsi disk config is not found, fall back to the original behavior.
		// This portal/iqn/iface is no longer referenced, log out.
		// Extract the portal and iqn from device path.
		bkpPortal = make([]string, 1)
		bkpPortal[0], iqn, err = extractPortalAndIqn(device)
		if err != nil {
			return err
		}
		arr := strings.Split(device, "-lun-")
		if len(arr) < 2 {
			return fmt.Errorf("failed to retrieve lun from mapPath: %v", mapPath)
		}
		lun = arr[1]
		// Extract the iface from the mountPath and use it to log out. If the iface
		// is not found, maintain the previous behavior to facilitate kubelet upgrade.
		// Logout may fail as no session may exist for the portal/IQN on the specified interface.
		iface, found = extractIface(mapPath)
	}
	portals := removeDuplicate(bkpPortal)
	if len(portals) == 0 {
		return fmt.Errorf("iscsi detach disk: failed to detach iscsi disk. Couldn't get connected portals from configurations")
	}

	devicePath := getDevByPath(portals[0], iqn, lun)
	klog.V(5).Infof("iscsi: devicePath: %s", devicePath)
	if _, err = os.Stat(devicePath); err != nil {
		return fmt.Errorf("failed to validate devicePath: %s", devicePath)
	}

	// Lock the target while we determine if we can safely log out or not
	c.plugin.targetLocks.LockKey(iqn)
	defer c.plugin.targetLocks.UnlockKey(iqn)

	// If device is no longer used, see if need to logout the target
	if isSessionBusy(c.iscsiDisk.plugin.host, portals[0], iqn) {
		return nil
	}

	// Detach a volume from kubelet node
	err = util.detachISCSIDisk(c.exec, portals, iqn, iface, volName, initiatorName, found)
	if err != nil {
		return fmt.Errorf("failed to finish detachISCSIDisk, err: %v", err)
	}
	return nil
}


func (util *ISCSIUtil) detachISCSIDisk(exec utilexec.Interface, portals []string, iqn, iface, volName, initiatorName string, found bool) error {
	for _, portal := range portals {
		logoutArgs := []string{"-m", "node", "-p", portal, "-T", iqn, "--logout"}
		deleteArgs := []string{"-m", "node", "-p", portal, "-T", iqn, "-o", "delete"}
		if found {
			logoutArgs = append(logoutArgs, []string{"-I", iface}...)
			deleteArgs = append(deleteArgs, []string{"-I", iface}...)
		}
		klog.Infof("iscsi: log out target %s iqn %s iface %s", portal, iqn, iface)
		out, err := exec.Command("iscsiadm", logoutArgs...).CombinedOutput()
		err = ignoreExitCodes(err, exit_ISCSI_ERR_NO_OBJS_FOUND, exit_ISCSI_ERR_SESS_NOT_FOUND)
		if err != nil {
			klog.Errorf("iscsi: failed to detach disk Error: %s", string(out))
			return err
		}
		// Delete the node record
		klog.Infof("iscsi: delete node record target %s iqn %s", portal, iqn)
		out, err = exec.Command("iscsiadm", deleteArgs...).CombinedOutput()
		err = ignoreExitCodes(err, exit_ISCSI_ERR_NO_OBJS_FOUND, exit_ISCSI_ERR_SESS_NOT_FOUND)
		if err != nil {
			klog.Errorf("iscsi: failed to delete node record Error: %s", string(out))
			return err
		}
	}
	// Delete the iface after all sessions have logged out
	// If the iface is not created via iscsi plugin, skip to delete
	if initiatorName != "" && found && iface == (portals[0]+":"+volName) {
		deleteArgs := []string{"-m", "iface", "-I", iface, "-o", "delete"}
		out, err := exec.Command("iscsiadm", deleteArgs...).CombinedOutput()
		err = ignoreExitCodes(err, exit_ISCSI_ERR_NO_OBJS_FOUND, exit_ISCSI_ERR_SESS_NOT_FOUND)
		if err != nil {
			klog.Errorf("iscsi: failed to delete iface Error: %s", string(out))
			return err
		}
	}

	return nil
}


func getDevByPath(portal, iqn, lun string) string {
	return "/dev/disk/by-path/ip-" + portal + "-iscsi-" + iqn + "-lun-" + lun
}


func extractTransportname(ifaceOutput string) (iscsiTransport string) {
	rexOutput := ifaceTransportNameRe.FindStringSubmatch(ifaceOutput)
	if rexOutput == nil {
		return ""
	}
	iscsiTransport = rexOutput[1]

	// While iface.transport_name is a required parameter, handle it being unspecified anyways
	if iscsiTransport == "<empty>" {
		iscsiTransport = "tcp"
	}
	return iscsiTransport
}


func extractDeviceAndPrefix(mntPath string) (string, string, error) {
	ind := strings.LastIndex(mntPath, "/")
	if ind < 0 {
		return "", "", fmt.Errorf("iscsi detach disk: malformatted mnt path: %s", mntPath)
	}
	device := mntPath[(ind + 1):]
	// strip -lun- from mount path
	ind = strings.LastIndex(mntPath, "-lun-")
	if ind < 0 {
		return "", "", fmt.Errorf("iscsi detach disk: malformatted mnt path: %s", mntPath)
	}
	prefix := mntPath[:ind]
	return device, prefix, nil
}


func extractIface(mntPath string) (string, bool) {
	reOutput := ifaceRe.FindStringSubmatch(mntPath)
	if len(reOutput) > 1 {
		return reOutput[1], true
	}

	return "", false
}


func extractPortalAndIqn(device string) (string, string, error) {
	ind1 := strings.Index(device, "-")
	if ind1 < 0 {
		return "", "", fmt.Errorf("iscsi detach disk: no portal in %s", device)
	}
	portal := device[0:ind1]
	ind2 := strings.Index(device, "iqn.")
	if ind2 < 0 {
		ind2 = strings.Index(device, "eui.")
	}
	if ind2 < 0 {
		return "", "", fmt.Errorf("iscsi detach disk: no iqn in %s", device)
	}
	ind := strings.LastIndex(device, "-lun-")
	iqn := device[ind2:ind]
	return portal, iqn, nil
}


// Remove duplicates or string
func removeDuplicate(s []string) []string {
	m := map[string]bool{}
	for _, v := range s {
		if v != "" && !m[v] {
			s[len(m)] = v
			m[v] = true
		}
	}
	s = s[:len(m)]
	return s
}


func parseIscsiadmShow(output string) (map[string]string, error) {
	params := make(map[string]string)
	slice := strings.Split(output, "\n")
	for _, line := range slice {
		if !strings.HasPrefix(line, "iface.") || strings.Contains(line, "<empty>") {
			continue
		}
		iface := strings.Fields(line)
		if len(iface) != 3 || iface[1] != "=" {
			return nil, fmt.Errorf("error: invalid iface setting: %v", iface)
		}
		// iscsi_ifacename is immutable once the iface is created
		if iface[0] == "iface.iscsi_ifacename" {
			continue
		}
		params[iface[0]] = iface[2]
	}
	return params, nil
}


func cloneIface(b iscsiDiskMounter) error {
	var lastErr error
	if b.InitIface == b.Iface {
		return fmt.Errorf("iscsi: cannot clone iface with same name: %s", b.InitIface)
	}
	// get pre-configured iface records
	out, err := execWithLog(b, "iscsiadm", "-m", "iface", "-I", b.InitIface, "-o", "show")
	if err != nil {
		lastErr = fmt.Errorf("iscsi: failed to show iface records: %s (%v)", out, err)
		return lastErr
	}
	// parse obtained records
	params, err := parseIscsiadmShow(out)
	if err != nil {
		lastErr = fmt.Errorf("iscsi: failed to parse iface records: %s (%v)", out, err)
		return lastErr
	}
	// update initiatorname
	params["iface.initiatorname"] = b.InitiatorName
	// create new iface
	out, err = execWithLog(b, "iscsiadm", "-m", "iface", "-I", b.Iface, "-o", "new")
	if err != nil {
		exit, ok := err.(utilexec.ExitError)
		if ok && exit.ExitStatus() == iscsiadmErrorSessExists {
			klog.Infof("iscsi: there is a session already logged in with iface %s", b.Iface)
		} else {
			lastErr = fmt.Errorf("iscsi: failed to create new iface: %s (%v)", out, err)
			return lastErr
		}
	}
	// Get and sort keys to maintain a stable iteration order
	var keys []string
	for k := range params {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	// update new iface records
	for _, key := range keys {
		_, err = execWithLog(b, "iscsiadm", "-m", "iface", "-I", b.Iface, "-o", "update", "-n", key, "-v", params[key])
		if err != nil {
			execWithLog(b, "iscsiadm", "-m", "iface", "-I", b.Iface, "-o", "delete")
			lastErr = fmt.Errorf("iscsi: failed to update iface records: %s (%v). iface(%s) will be used", out, err, b.InitIface)
			break
		}
	}
	return lastErr
}


// isSessionBusy determines if the iSCSI session is busy by counting both FS and block volumes in use.
func isSessionBusy(host volume.VolumeHost, portal, iqn string) bool {
	fsDir := host.GetPluginDir(iscsiPluginName)
	countFS, err := getVolCount(fsDir, portal, iqn)
	if err != nil {
		klog.Errorf("iscsi: could not determine FS volumes in use: %v", err)
		return true
	}

	blockDir := host.GetVolumeDevicePluginDir(iscsiPluginName)
	countBlock, err := getVolCount(blockDir, portal, iqn)
	if err != nil {
		klog.Errorf("iscsi: could not determine block volumes in use: %v", err)
		return true
	}

	return countFS+countBlock > 1
}


// getVolCount returns the number of volumes in use by the kubelet.
// It does so by counting the number of directories prefixed by the given portal and IQN.
func getVolCount(dir, portal, iqn string) (int, error) {
	// For FileSystem volumes, the topmost dirs are named after the ifaces, e.g., iface-default or iface-127.0.0.1:3260:pv0.
	// For Block volumes, the default topmost dir is volumeDevices.
	contents, err := ioutil.ReadDir(dir)
	if err != nil {
		if os.IsNotExist(err) {
			return 0, nil
		}
		return 0, err
	}

	// Inside each iface dir, we look for volume dirs prefixed by the given
	// portal + iqn, e.g., 127.0.0.1:3260-iqn.2003-01.io.k8s:e2e.volume-1-lun-2
	var counter int
	for _, c := range contents {
		if !c.IsDir() || c.Name() == config.DefaultKubeletVolumeDevicesDirName {
			continue
		}

		mounts, err := ioutil.ReadDir(filepath.Join(dir, c.Name()))
		if err != nil {
			return 0, err
		}

		for _, m := range mounts {
			volumeMount := m.Name()
			prefix := portal + "-" + iqn
			if strings.HasPrefix(volumeMount, prefix) {
				counter++
			}
		}
	}

	return counter, nil
}


func ignoreExitCodes(err error, ignoredExitCodes ...int) error {
	exitError, ok := err.(utilexec.ExitError)
	if !ok {
		return err
	}
	for _, code := range ignoredExitCodes {
		if exitError.ExitStatus() == code {
			klog.V(4).Infof("ignored iscsiadm exit code %d", code)
			return nil
		}
	}
	return err
}


func execWithLog(b iscsiDiskMounter, cmd string, args ...string) (string, error) {
	start := time.Now()
	out, err := b.exec.Command(cmd, args...).CombinedOutput()
	if klogV := klog.V(5); klogV.Enabled() {
		d := time.Since(start)
		klogV.Infof("Executed %s %v in %v, err: %v", cmd, args, d, err)
		klogV.Infof("Output: %s", string(out))
	}
	return string(out), err
}


// ProbeVolumePlugins is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&localVolumePlugin{}}
}


func (plugin *localVolumePlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	plugin.volumeLocks = keymutex.NewHashed(0)
	plugin.recorder = host.GetEventRecorder()
	return nil
}


func (plugin *localVolumePlugin) GetPluginName() string {
	return localVolumePluginName
}


func (plugin *localVolumePlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	// This volume is only supported as a PersistentVolumeSource, so the PV name is unique
	return spec.Name(), nil
}


func (plugin *localVolumePlugin) CanSupport(spec *volume.Spec) bool {
	// This volume is only supported as a PersistentVolumeSource
	return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.Local != nil)
}


func (plugin *localVolumePlugin) RequiresRemount(spec *volume.Spec) bool {
	return false
}


func (plugin *localVolumePlugin) SupportsMountOption() bool {
	return true
}


func (plugin *localVolumePlugin) SupportsBulkVolumeVerification() bool {
	return false
}


func (plugin *localVolumePlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return false, nil
}


func (plugin *localVolumePlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
	// The current meaning of AccessMode is how many nodes can attach to it, not how many pods can mount it
	return []v1.PersistentVolumeAccessMode{
		v1.ReadWriteOnce,
	}
}


func getVolumeSource(spec *volume.Spec) (*v1.LocalVolumeSource, bool, error) {
	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.Local != nil {
		return spec.PersistentVolume.Spec.Local, spec.ReadOnly, nil
	}

	return nil, false, fmt.Errorf("Spec does not reference a Local volume type")
}


func (plugin *localVolumePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
	_, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}

	globalLocalPath, err := plugin.getGlobalLocalPath(spec)
	if err != nil {
		return nil, err
	}

	kvh, ok := plugin.host.(volume.KubeletVolumeHost)
	if !ok {
		return nil, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
	}

	return &localVolumeMounter{
		localVolume: &localVolume{
			pod:             pod,
			podUID:          pod.UID,
			volName:         spec.Name(),
			mounter:         plugin.host.GetMounter(plugin.GetPluginName()),
			hostUtil:        kvh.GetHostUtil(),
			plugin:          plugin,
			globalPath:      globalLocalPath,
			MetricsProvider: volume.NewMetricsStatFS(plugin.host.GetPodVolumeDir(pod.UID, utilstrings.EscapeQualifiedName(localVolumePluginName), spec.Name())),
		},
		mountOptions: util.MountOptionFromSpec(spec),
		readOnly:     readOnly,
	}, nil

}


func (plugin *localVolumePlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	return &localVolumeUnmounter{
		localVolume: &localVolume{
			podUID:  podUID,
			volName: volName,
			mounter: plugin.host.GetMounter(plugin.GetPluginName()),
			plugin:  plugin,
		},
	}, nil
}


func (plugin *localVolumePlugin) NewBlockVolumeMapper(spec *volume.Spec, pod *v1.Pod,
	_ volume.VolumeOptions) (volume.BlockVolumeMapper, error) {
	volumeSource, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}

	mapper := &localVolumeMapper{
		localVolume: &localVolume{
			podUID:     pod.UID,
			volName:    spec.Name(),
			globalPath: volumeSource.Path,
			plugin:     plugin,
		},
		readOnly: readOnly,
	}

	blockPath, err := mapper.GetGlobalMapPath(spec)
	if err != nil {
		return nil, fmt.Errorf("failed to get device path: %v", err)
	}
	mapper.MetricsProvider = volume.NewMetricsBlock(filepath.Join(blockPath, string(pod.UID)))

	return mapper, nil
}


func (plugin *localVolumePlugin) NewBlockVolumeUnmapper(volName string,
	podUID types.UID) (volume.BlockVolumeUnmapper, error) {
	return &localVolumeUnmapper{
		localVolume: &localVolume{
			podUID:  podUID,
			volName: volName,
			plugin:  plugin,
		},
	}, nil
}


// TODO: check if no path and no topology constraints are ok
func (plugin *localVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
	fs := v1.PersistentVolumeFilesystem
	// The main purpose of reconstructed volume is to clean unused mount points
	// and directories.
	// For filesystem volume with directory source, no global mount path is
	// needed to clean. Empty path is ok.
	// For filesystem volume with block source, we should resolve to its device
	// path if global mount path exists.
	var path string
	mounter := plugin.host.GetMounter(plugin.GetPluginName())
	refs, err := mounter.GetMountRefs(mountPath)
	if err != nil {
		return volume.ReconstructedVolume{}, err
	}
	baseMountPath := plugin.generateBlockDeviceBaseGlobalPath()
	for _, ref := range refs {
		if mount.PathWithinBase(ref, baseMountPath) {
			// If the global mount for block device exists, the source is block
			// device.
			// The resolved device path may not be the exact same as path in
			// local PV object if symbolic link is used. However, it's the true
			// source and can be used in reconstructed volume.
			path, _, err = mount.GetDeviceNameFromMount(mounter, ref)
			if err != nil {
				return volume.ReconstructedVolume{}, err
			}
			klog.V(4).Infof("local: reconstructing volume %q (pod volume mount: %q) with device %q", volumeName, mountPath, path)
			break
		}
	}
	localVolume := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: volumeName,
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeSource: v1.PersistentVolumeSource{
				Local: &v1.LocalVolumeSource{
					Path: path,
				},
			},
			VolumeMode: &fs,
		},
	}
	return volume.ReconstructedVolume{
		Spec: volume.NewSpecFromPersistentVolume(localVolume, false),
	}, nil
}


func (plugin *localVolumePlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName,
	mapPath string) (*volume.Spec, error) {
	block := v1.PersistentVolumeBlock

	localVolume := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: volumeName,
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeSource: v1.PersistentVolumeSource{
				Local: &v1.LocalVolumeSource{
					// Not needed because we don't need to detach local device from the host.
					Path: "",
				},
			},
			VolumeMode: &block,
		},
	}

	return volume.NewSpecFromPersistentVolume(localVolume, false), nil
}


func (plugin *localVolumePlugin) generateBlockDeviceBaseGlobalPath() string {
	return filepath.Join(plugin.host.GetPluginDir(localVolumePluginName), util.MountsInGlobalPDPath)
}


func (plugin *localVolumePlugin) getGlobalLocalPath(spec *volume.Spec) (string, error) {
	if spec.PersistentVolume.Spec.Local == nil || len(spec.PersistentVolume.Spec.Local.Path) == 0 {
		return "", fmt.Errorf("local volume source is nil or local path is not set")
	}

	kvh, ok := plugin.host.(volume.KubeletVolumeHost)
	if !ok {
		return "", fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
	}

	fileType, err := kvh.GetHostUtil().GetFileType(spec.PersistentVolume.Spec.Local.Path)
	if err != nil {
		return "", err
	}
	switch fileType {
	case hostutil.FileTypeDirectory:
		return spec.PersistentVolume.Spec.Local.Path, nil
	case hostutil.FileTypeBlockDev:
		return filepath.Join(plugin.generateBlockDeviceBaseGlobalPath(), spec.Name()), nil
	default:
		return "", fmt.Errorf("only directory and block device are supported")
	}
}


func (plugin *localVolumePlugin) CanDeviceMount(spec *volume.Spec) (bool, error) {
	return true, nil
}


func (plugin *localVolumePlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
	kvh, ok := plugin.host.(volume.KubeletVolumeHost)
	if !ok {
		return nil, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
	}
	return &deviceMounter{
		plugin:   plugin,
		mounter:  util.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host),
		hostUtil: kvh.GetHostUtil(),
	}, nil
}


func (dm *deviceMounter) mountLocalBlockDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error {
	klog.V(4).Infof("local: mounting device %s to %s", devicePath, deviceMountPath)
	notMnt, err := dm.mounter.IsLikelyNotMountPoint(deviceMountPath)
	if err != nil {
		if os.IsNotExist(err) {
			if err := os.MkdirAll(deviceMountPath, 0750); err != nil {
				return err
			}
			notMnt = true
		} else {
			return err
		}
	}
	if !notMnt {
		return nil
	}
	fstype, err := getVolumeSourceFSType(spec)
	if err != nil {
		return err
	}

	ro, err := getVolumeSourceReadOnly(spec)
	if err != nil {
		return err
	}
	options := []string{}
	if ro {
		options = append(options, "ro")
	}
	mountOptions := util.MountOptionFromSpec(spec, options...)
	err = dm.mounter.FormatAndMount(devicePath, deviceMountPath, fstype, mountOptions)
	if err != nil {
		if rmErr := os.Remove(deviceMountPath); rmErr != nil {
			klog.Warningf("local: failed to remove %s: %v", deviceMountPath, rmErr)
		}
		return fmt.Errorf("local: failed to mount device %s at %s (fstype: %s), error %w", devicePath, deviceMountPath, fstype, err)
	}
	klog.V(3).Infof("local: successfully mount device %s at %s (fstype: %s)", devicePath, deviceMountPath, fstype)
	return nil
}


func (dm *deviceMounter) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string, _ volume.DeviceMounterArgs) error {
	if spec.PersistentVolume.Spec.Local == nil || len(spec.PersistentVolume.Spec.Local.Path) == 0 {
		return fmt.Errorf("local volume source is nil or local path is not set")
	}
	fileType, err := dm.hostUtil.GetFileType(spec.PersistentVolume.Spec.Local.Path)
	if err != nil {
		return err
	}

	switch fileType {
	case hostutil.FileTypeBlockDev:
		// local volume plugin does not implement AttachableVolumePlugin interface, so set devicePath to Path in PV spec directly
		return dm.mountLocalBlockDevice(spec, spec.PersistentVolume.Spec.Local.Path, deviceMountPath)
	case hostutil.FileTypeDirectory:
		// if the given local volume path is of already filesystem directory, return directly
		return nil
	default:
		return fmt.Errorf("only directory and block device are supported")
	}
}


func (plugin *localVolumePlugin) RequiresFSResize() bool {
	return true
}


func (plugin *localVolumePlugin) NodeExpand(resizeOptions volume.NodeResizeOptions) (bool, error) {
	fsVolume, err := util.CheckVolumeModeFilesystem(resizeOptions.VolumeSpec)
	if err != nil {
		return false, fmt.Errorf("error checking VolumeMode: %v", err)
	}
	if !fsVolume {
		return true, nil
	}

	localDevicePath := resizeOptions.VolumeSpec.PersistentVolume.Spec.Local.Path

	kvh, ok := plugin.host.(volume.KubeletVolumeHost)
	if !ok {
		return false, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
	}

	fileType, err := kvh.GetHostUtil().GetFileType(localDevicePath)
	if err != nil {
		return false, err
	}

	switch fileType {
	case hostutil.FileTypeBlockDev:
		_, err = util.GenericResizeFS(plugin.host, plugin.GetPluginName(), localDevicePath, resizeOptions.DeviceMountPath)
		if err != nil {
			return false, err
		}
		return true, nil
	case hostutil.FileTypeDirectory:
		// if the given local volume path is of already filesystem directory, return directly because
		// we do not want to prevent mount operation from succeeding.
		klog.InfoS("Expansion of directory based local volumes is NO-OP", "localVolumePath", localDevicePath)
		return true, nil
	default:
		return false, fmt.Errorf("only directory and block device are supported")
	}
}


func getVolumeSourceFSType(spec *volume.Spec) (string, error) {
	if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.Local != nil {
		if spec.PersistentVolume.Spec.Local.FSType != nil {
			return *spec.PersistentVolume.Spec.Local.FSType, nil
		}
		// if the FSType is not set in local PV spec, setting it to default ("ext4")
		return defaultFSType, nil
	}

	return "", fmt.Errorf("spec does not reference a Local volume type")
}


func getVolumeSourceReadOnly(spec *volume.Spec) (bool, error) {
	if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.Local != nil {
		// local volumes used as a PersistentVolume gets the ReadOnly flag indirectly through
		// the persistent-claim volume used to mount the PV
		return spec.ReadOnly, nil
	}

	return false, fmt.Errorf("spec does not reference a Local volume type")
}


func (dm *deviceMounter) GetDeviceMountPath(spec *volume.Spec) (string, error) {
	return dm.plugin.getGlobalLocalPath(spec)
}


func (plugin *localVolumePlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
	return &deviceMounter{
		plugin:  plugin,
		mounter: util.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host),
	}, nil
}


func (plugin *localVolumePlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
	mounter := plugin.host.GetMounter(plugin.GetPluginName())
	return mounter.GetMountRefs(deviceMountPath)
}


func (dm *deviceMounter) UnmountDevice(deviceMountPath string) error {
	// If the local PV is a block device,
	// The deviceMountPath is generated to the format like :/var/lib/kubelet/plugins/kubernetes.io/local-volume/mounts/localpv.spec.Name;
	// If it is a filesystem directory, then the deviceMountPath is set directly to pvSpec.Local.Path
	// We only need to unmount block device here, so we need to check if the deviceMountPath passed here
	// has base mount path: /var/lib/kubelet/plugins/kubernetes.io/local-volume/mounts
	basemountPath := dm.plugin.generateBlockDeviceBaseGlobalPath()
	if mount.PathWithinBase(deviceMountPath, basemountPath) {
		return mount.CleanupMountPoint(deviceMountPath, dm.mounter, false)
	}

	return nil
}


func (l *localVolume) GetPath() string

func (l *localVolume) GetPath() string {
	return l.plugin.host.GetPodVolumeDir(l.podUID, utilstrings.EscapeQualifiedName(localVolumePluginName), l.volName)
}


func (m *localVolumeMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:       m.readOnly,
		Managed:        !m.readOnly,
		SELinuxRelabel: true,
	}
}


// SetUp bind mounts the directory to the volume path
func (m *localVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error {
	return m.SetUpAt(m.GetPath(), mounterArgs)
}


// SetUpAt bind mounts the directory to the volume path and sets up volume ownership
func (m *localVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	m.plugin.volumeLocks.LockKey(m.globalPath)
	defer m.plugin.volumeLocks.UnlockKey(m.globalPath)

	if m.globalPath == "" {
		return fmt.Errorf("LocalVolume volume %q path is empty", m.volName)
	}

	err := validation.ValidatePathNoBacksteps(m.globalPath)
	if err != nil {
		return fmt.Errorf("invalid path: %s %v", m.globalPath, err)
	}

	notMnt, err := mount.IsNotMountPoint(m.mounter, dir)
	klog.V(4).Infof("LocalVolume mount setup: PodDir(%s) VolDir(%s) Mounted(%t) Error(%v), ReadOnly(%t)", dir, m.globalPath, !notMnt, err, m.readOnly)
	if err != nil && !os.IsNotExist(err) {
		klog.Errorf("cannot validate mount point: %s %v", dir, err)
		return err
	}

	if !notMnt {
		return nil
	}
	refs, err := m.mounter.GetMountRefs(m.globalPath)
	if mounterArgs.FsGroup != nil {
		if err != nil {
			klog.Errorf("cannot collect mounting information: %s %v", m.globalPath, err)
			return err
		}

		// Only count mounts from other pods
		refs = m.filterPodMounts(refs)
		if len(refs) > 0 {
			fsGroupNew := int64(*mounterArgs.FsGroup)
			_, fsGroupOld, err := m.hostUtil.GetOwner(m.globalPath)
			if err != nil {
				return fmt.Errorf("failed to check fsGroup for %s (%v)", m.globalPath, err)
			}
			if fsGroupNew != fsGroupOld {
				m.plugin.recorder.Eventf(m.pod, v1.EventTypeWarning, events.WarnAlreadyMountedVolume, "The requested fsGroup is %d, but the volume %s has GID %d. The volume may not be shareable.", fsGroupNew, m.volName, fsGroupOld)
			}
		}

	}

	if runtime.GOOS != "windows" {
		// skip below MkdirAll for windows since the "bind mount" logic is implemented differently in mount_wiondows.go
		if err := os.MkdirAll(dir, 0750); err != nil {
			klog.Errorf("mkdir failed on disk %s (%v)", dir, err)
			return err
		}
	}
	// Perform a bind mount to the full path to allow duplicate mounts of the same volume.
	options := []string{"bind"}
	if m.readOnly {
		options = append(options, "ro")
	}
	mountOptions := util.JoinMountOptions(options, m.mountOptions)

	klog.V(4).Infof("attempting to mount %s", dir)
	globalPath := util.MakeAbsolutePath(runtime.GOOS, m.globalPath)
	err = m.mounter.MountSensitiveWithoutSystemd(globalPath, dir, "", mountOptions, nil)
	if err != nil {
		klog.Errorf("Mount of volume %s failed: %v", dir, err)
		notMnt, mntErr := mount.IsNotMountPoint(m.mounter, dir)
		if mntErr != nil {
			klog.Errorf("IsNotMountPoint check failed: %v", mntErr)
			return err
		}
		if !notMnt {
			if mntErr = m.mounter.Unmount(dir); mntErr != nil {
				klog.Errorf("Failed to unmount: %v", mntErr)
				return err
			}
			notMnt, mntErr = mount.IsNotMountPoint(m.mounter, dir)
			if mntErr != nil {
				klog.Errorf("IsNotMountPoint check failed: %v", mntErr)
				return err
			}
			if !notMnt {
				// This is very odd, we don't expect it.  We'll try again next sync loop.
				klog.Errorf("%s is still mounted, despite call to unmount().  Will try again next sync loop.", dir)
				return err
			}
		}
		if rmErr := os.Remove(dir); rmErr != nil {
			klog.Warningf("failed to remove %s: %v", dir, rmErr)
		}
		return err
	}
	if !m.readOnly {
		// Volume owner will be written only once on the first volume mount
		if len(refs) == 0 {
			return volume.SetVolumeOwnership(m, dir, mounterArgs.FsGroup, mounterArgs.FSGroupChangePolicy, util.FSGroupCompleteHook(m.plugin, nil))
		}
	}
	return nil
}


// filterPodMounts only returns mount paths inside the kubelet pod directory
func (m *localVolumeMounter) filterPodMounts(refs []string) []string {
	filtered := []string{}
	for _, r := range refs {
		if strings.HasPrefix(r, m.plugin.host.GetPodsDir()+string(os.PathSeparator)) {
			filtered = append(filtered, r)
		}
	}
	return filtered
}


// TearDown unmounts the bind mount
func (u *localVolumeUnmounter) TearDown() error {
	return u.TearDownAt(u.GetPath())
}


// TearDownAt unmounts the bind mount
func (u *localVolumeUnmounter) TearDownAt(dir string) error {
	klog.V(4).Infof("Unmounting volume %q at path %q\n", u.volName, dir)
	return mount.CleanupMountPoint(dir, u.mounter, true) /* extensiveMountPointCheck = true */
}


// SetUpDevice prepares the volume to the node by the plugin specific way.
func (m *localVolumeMapper) SetUpDevice() (string, error) {
	return "", nil
}


// MapPodDevice provides physical device path for the local PV.
func (m *localVolumeMapper) MapPodDevice() (string, error) {
	globalPath := util.MakeAbsolutePath(runtime.GOOS, m.globalPath)
	klog.V(4).Infof("MapPodDevice returning path %s", globalPath)
	return globalPath, nil
}


// GetStagingPath returns
func (m *localVolumeMapper) GetStagingPath() string {
	return ""
}


// SupportsMetrics returns true for SupportsMetrics as it initializes the
// MetricsProvider.
func (m *localVolumeMapper) SupportsMetrics() bool {
	return true
}


// GetGlobalMapPath returns global map path and error.
// path: plugins/kubernetes.io/kubernetes.io/local-volume/volumeDevices/{volumeName}
func (l *localVolume) GetGlobalMapPath(spec *volume.Spec) (string, error) {
	return filepath.Join(l.plugin.host.GetVolumeDevicePluginDir(utilstrings.EscapeQualifiedName(localVolumePluginName)),
		l.volName), nil
}


// GetPodDeviceMapPath returns pod device map path and volume name.
// path: pods/{podUid}/volumeDevices/kubernetes.io~local-volume
// volName: local-pv-ff0d6d4
func (l *localVolume) GetPodDeviceMapPath() (string, string) {
	return l.plugin.host.GetPodVolumeDeviceDir(l.podUID,
		utilstrings.EscapeQualifiedName(localVolumePluginName)), l.volName
}


// NewMetricsStatfs creates a new metricsBlock with the device node of the
// Volume.
func NewMetricsBlock(device string) MetricsProvider {
	return &metricsBlock{device}
}


// See MetricsProvider.GetMetrics
// GetMetrics detects the size of the BlockMode volume for the device node
// where the Volume is attached.
//
// Note that only the capacity of the device can be detected with standard
// tools. Storage systems may have more information that they can provide by
// going through specialized APIs.
func (mb *metricsBlock) GetMetrics() (*Metrics, error) {
	startTime := time.Now()
	defer servermetrics.CollectVolumeStatCalDuration("block", startTime)

	// TODO: Windows does not yet support VolumeMode=Block
	if runtime.GOOS == "windows" {
		return nil, NewNotImplementedError("Windows does not support Block volumes")
	}

	metrics := &Metrics{Time: metav1.Now()}
	if mb.device == "" {
		return metrics, NewNoPathDefinedError()
	}

	err := mb.getBlockInfo(metrics)
	if err != nil {
		return metrics, err
	}

	return metrics, nil
}


// getBlockInfo fetches metrics.Capacity by opening the device and seeking to
// the end.
func (mb *metricsBlock) getBlockInfo(metrics *Metrics) error {
	dev, err := os.Open(mb.device)
	if err != nil {
		return fmt.Errorf("unable to open device %q: %w", mb.device, err)
	}
	defer dev.Close()

	end, err := dev.Seek(0, io.SeekEnd)
	if err != nil {
		return fmt.Errorf("failed to detect size of %q: %w", mb.device, err)
	}

	metrics.Capacity = resource.NewQuantity(end, resource.BinarySI)

	return nil
}


// NewCachedMetrics creates a new cachedMetrics wrapping another
// MetricsProvider and caching the results.
func NewCachedMetrics(provider MetricsProvider) MetricsProvider {
	return &cachedMetrics{wrapped: provider}
}


// GetMetrics runs the wrapped metrics provider's GetMetrics method once and
// caches the result. Will not cache result if there is an error.
// See MetricsProvider.GetMetrics
func (md *cachedMetrics) GetMetrics() (*Metrics, error) {
	md.once.cache(func() error {
		md.resultMetrics, md.resultError = md.wrapped.GetMetrics()
		return md.resultError
	})
	return md.resultMetrics, md.resultError
}


// Copied from sync.Once but we don't want to cache the results if there is an
// error
func (o *cacheOnce) cache(f func() error) {
	if atomic.LoadUint32(&o.done) == 1 {
		return
	}
	// Slow-path.
	o.m.Lock()
	defer o.m.Unlock()
	if o.done == 0 {
		err := f()
		if err == nil {
			atomic.StoreUint32(&o.done, 1)
		}
	}
}


// NewMetricsDu creates a new metricsDu with the Volume path.
func NewMetricsDu(path string) MetricsProvider {
	return &metricsDu{path}
}


// GetMetrics calculates the volume usage and device free space by executing "du"
// and gathering filesystem info for the Volume path.
// See MetricsProvider.GetMetrics
func (md *metricsDu) GetMetrics() (*Metrics, error) {
	metrics := &Metrics{Time: metav1.Now()}
	if md.path == "" {
		return metrics, NewNoPathDefinedError()
	}

	err := md.getDiskUsage(metrics)
	if err != nil {
		return metrics, err
	}

	err = md.getFsInfo(metrics)
	if err != nil {
		return metrics, err
	}

	return metrics, nil
}


// getDiskUsage writes metrics.Used and metric.InodesUsed from fs.DiskUsage
func (md *metricsDu) getDiskUsage(metrics *Metrics) error {
	usage, err := fs.DiskUsage(md.path)
	if err != nil {
		return err
	}
	metrics.Used = resource.NewQuantity(usage.Bytes, resource.BinarySI)
	metrics.InodesUsed = resource.NewQuantity(usage.Inodes, resource.BinarySI)
	return nil
}


// getFsInfo writes metrics.Capacity and metrics.Available from the filesystem
// info
func (md *metricsDu) getFsInfo(metrics *Metrics) error {
	available, capacity, _, inodes, inodesFree, _, err := fs.Info(md.path)
	if err != nil {
		return NewFsInfoFailedError(err)
	}
	metrics.Available = resource.NewQuantity(available, resource.BinarySI)
	metrics.Capacity = resource.NewQuantity(capacity, resource.BinarySI)
	metrics.Inodes = resource.NewQuantity(inodes, resource.BinarySI)
	metrics.InodesFree = resource.NewQuantity(inodesFree, resource.BinarySI)
	return nil
}


// NewNotSupportedError creates a new MetricsError with code NotSupported.
func NewNotSupportedError() *MetricsError {
	return &MetricsError{
		Code: ErrCodeNotSupported,
		Msg:  "metrics are not supported for MetricsNil Volumes",
	}
}


// NewNotImplementedError creates a new MetricsError with code NotSupported.
func NewNotImplementedError(reason string) *MetricsError {
	return &MetricsError{
		Code: ErrCodeNotSupported,
		Msg:  fmt.Sprintf("metrics support is not implemented: %s", reason),
	}
}


// NewNotSupportedErrorWithDriverName creates a new MetricsError with code NotSupported.
// driver name is added to the error message.
func NewNotSupportedErrorWithDriverName(name string) *MetricsError {
	return &MetricsError{
		Code: ErrCodeNotSupported,
		Msg:  fmt.Sprintf("metrics are not supported for %s volumes", name),
	}
}


// NewNoPathDefinedError creates a new MetricsError with code NoPathDefined.
func NewNoPathDefinedError() *MetricsError {
	return &MetricsError{
		Code: ErrCodeNoPathDefined,
		Msg:  "no path defined for disk usage metrics.",
	}
}


// NewFsInfoFailedError creates a new MetricsError with code FsInfoFailed.
func NewFsInfoFailedError(err error) *MetricsError {
	return &MetricsError{
		Code: ErrCodeFsInfoFailed,
		Msg:  fmt.Sprintf("failed to get FsInfo due to error %v", err),
	}
}


func (e *MetricsError) Error() string

func (e *MetricsError) Error() string {
	return e.Msg
}


func IsNotSupported(err error) bool

// IsNotSupported returns true if and only if err is "key" not found error.
func IsNotSupported(err error) bool {
	return isErrCode(err, ErrCodeNotSupported)
}


func isErrCode(err error, code int) bool {
	if err == nil {
		return false
	}
	if e, ok := err.(*MetricsError); ok {
		return e.Code == code
	}
	return false
}


// SupportsMetrics returns false for the MetricsNil type.
func (*MetricsNil) SupportsMetrics() bool {
	return false
}


// GetMetrics returns an empty Metrics and an error.
// See MetricsProvider.GetMetrics
func (*MetricsNil) GetMetrics() (*Metrics, error) {
	return &Metrics{}, NewNotSupportedError()
}


// NewMetricsStatfs creates a new metricsStatFS with the Volume path.
func NewMetricsStatFS(path string) MetricsProvider {
	return &metricsStatFS{path}
}


// See MetricsProvider.GetMetrics
// GetMetrics calculates the volume usage and device free space by executing "du"
// and gathering filesystem info for the Volume path.
func (md *metricsStatFS) GetMetrics() (*Metrics, error) {
	startTime := time.Now()
	defer servermetrics.CollectVolumeStatCalDuration("statfs", startTime)

	metrics := &Metrics{Time: metav1.Now()}
	if md.path == "" {
		return metrics, NewNoPathDefinedError()
	}

	err := md.getFsInfo(metrics)
	if err != nil {
		return metrics, err
	}

	return metrics, nil
}


// getFsInfo writes metrics.Capacity, metrics.Used and metrics.Available from the filesystem info
func (md *metricsStatFS) getFsInfo(metrics *Metrics) error {
	available, capacity, usage, inodes, inodesFree, inodesUsed, err := fs.Info(md.path)
	if err != nil {
		return NewFsInfoFailedError(err)
	}
	metrics.Available = resource.NewQuantity(available, resource.BinarySI)
	metrics.Capacity = resource.NewQuantity(capacity, resource.BinarySI)
	metrics.Used = resource.NewQuantity(usage, resource.BinarySI)
	metrics.Inodes = resource.NewQuantity(inodes, resource.BinarySI)
	metrics.InodesFree = resource.NewQuantity(inodesFree, resource.BinarySI)
	metrics.InodesUsed = resource.NewQuantity(inodesUsed, resource.BinarySI)
	return nil
}


func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
	return host.GetPodVolumeDir(uid, utilstrings.EscapeQualifiedName(nfsPluginName), volName)
}


// ProbeVolumePlugins is the primary entrypoint for volume plugins.
// This is the primary entrypoint for volume plugins.
// The volumeConfig arg provides the ability to configure recycler behavior.  It is implemented as a pointer to allow nils.
// The nfsPlugin is used to store the volumeConfig and give it, when needed, to the func that creates NFS Recyclers.
// Tests that exercise recycling should not use this func but instead use ProbeRecyclablePlugins() to override default behavior.
func ProbeVolumePlugins(volumeConfig volume.VolumeConfig) []volume.VolumePlugin {
	return []volume.VolumePlugin{
		&nfsPlugin{
			host:   nil,
			config: volumeConfig,
		},
	}
}


func (plugin *nfsPlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	return nil
}


func (plugin *nfsPlugin) GetPluginName() string {
	return nfsPluginName
}


func (plugin *nfsPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf(
		"%v/%v",
		volumeSource.Server,
		volumeSource.Path), nil
}


func (plugin *nfsPlugin) CanSupport(spec *volume.Spec) bool {
	return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.NFS != nil) ||
		(spec.Volume != nil && spec.Volume.NFS != nil)
}


func (plugin *nfsPlugin) RequiresRemount(spec *volume.Spec) bool {
	return false
}


func (plugin *nfsPlugin) SupportsMountOption() bool {
	return true
}


func (plugin *nfsPlugin) SupportsBulkVolumeVerification() bool {
	return false
}


func (plugin *nfsPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return false, nil
}


func (plugin *nfsPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
	return []v1.PersistentVolumeAccessMode{
		v1.ReadWriteOnce,
		v1.ReadOnlyMany,
		v1.ReadWriteMany,
	}
}


func (plugin *nfsPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
	return plugin.newMounterInternal(spec, pod, plugin.host.GetMounter(plugin.GetPluginName()))
}


func (plugin *nfsPlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod, mounter mount.Interface) (volume.Mounter, error) {
	source, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}
	return &nfsMounter{
		nfs: &nfs{
			volName:         spec.Name(),
			mounter:         mounter,
			pod:             pod,
			plugin:          plugin,
			MetricsProvider: volume.NewMetricsStatFS(getPath(pod.UID, spec.Name(), plugin.host)),
		},
		server:       getServerFromSource(source),
		exportPath:   source.Path,
		readOnly:     readOnly,
		mountOptions: util.MountOptionFromSpec(spec),
	}, nil
}


func (plugin *nfsPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter(plugin.GetPluginName()))
}


func (plugin *nfsPlugin) newUnmounterInternal(volName string, podUID types.UID, mounter mount.Interface) (volume.Unmounter, error) {
	return &nfsUnmounter{&nfs{
		volName:         volName,
		mounter:         mounter,
		pod:             &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: podUID}},
		plugin:          plugin,
		MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)),
	}}, nil
}


// Recycle recycles/scrubs clean an NFS volume.
// Recycle blocks until the pod has completed or any error occurs.
func (plugin *nfsPlugin) Recycle(pvName string, spec *volume.Spec, eventRecorder recyclerclient.RecycleEventRecorder) error {
	if spec.PersistentVolume == nil || spec.PersistentVolume.Spec.NFS == nil {
		return fmt.Errorf("spec.PersistentVolumeSource.NFS is nil")
	}

	pod := plugin.config.RecyclerPodTemplate
	timeout := util.CalculateTimeoutForVolume(plugin.config.RecyclerMinimumTimeout, plugin.config.RecyclerTimeoutIncrement, spec.PersistentVolume)
	// overrides
	pod.Spec.ActiveDeadlineSeconds = &timeout
	pod.GenerateName = "pv-recycler-nfs-"
	pod.Spec.Volumes[0].VolumeSource = v1.VolumeSource{
		NFS: &v1.NFSVolumeSource{
			Server: spec.PersistentVolume.Spec.NFS.Server,
			Path:   spec.PersistentVolume.Spec.NFS.Path,
		},
	}
	return recyclerclient.RecycleVolumeByWatchingPodUntilCompletion(pvName, pod, plugin.host.GetKubeClient(), eventRecorder)
}


func (plugin *nfsPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
	nfsVolume := &v1.Volume{
		Name: volumeName,
		VolumeSource: v1.VolumeSource{
			NFS: &v1.NFSVolumeSource{
				Path: volumeName,
			},
		},
	}
	return volume.ReconstructedVolume{
		Spec: volume.NewSpecFromVolume(nfsVolume),
	}, nil
}


func (nfsVolume *nfs) GetPath() string

func (nfsVolume *nfs) GetPath() string {
	name := nfsPluginName
	return nfsVolume.plugin.host.GetPodVolumeDir(nfsVolume.pod.UID, utilstrings.EscapeQualifiedName(name), nfsVolume.volName)
}


func (nfsMounter *nfsMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:       nfsMounter.readOnly,
		Managed:        false,
		SELinuxRelabel: false,
	}
}


// SetUp attaches the disk and bind mounts to the volume path.
func (nfsMounter *nfsMounter) SetUp(mounterArgs volume.MounterArgs) error {
	return nfsMounter.SetUpAt(nfsMounter.GetPath(), mounterArgs)
}


func (nfsMounter *nfsMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	notMnt, err := mount.IsNotMountPoint(nfsMounter.mounter, dir)
	klog.V(4).Infof("NFS mount set up: %s %v %v", dir, !notMnt, err)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	if !notMnt {
		return nil
	}
	if err := os.MkdirAll(dir, 0750); err != nil {
		return err
	}
	source := fmt.Sprintf("%s:%s", nfsMounter.server, nfsMounter.exportPath)
	options := []string{}
	if nfsMounter.readOnly {
		options = append(options, "ro")
	}
	mountOptions := util.JoinMountOptions(nfsMounter.mountOptions, options)
	err = nfsMounter.mounter.MountSensitiveWithoutSystemd(source, dir, "nfs", mountOptions, nil)
	if err != nil {
		notMnt, mntErr := mount.IsNotMountPoint(nfsMounter.mounter, dir)
		if mntErr != nil {
			klog.Errorf("IsNotMountPoint check failed: %v", mntErr)
			return err
		}
		if !notMnt {
			if mntErr = nfsMounter.mounter.Unmount(dir); mntErr != nil {
				klog.Errorf("Failed to unmount: %v", mntErr)
				return err
			}
			notMnt, mntErr := mount.IsNotMountPoint(nfsMounter.mounter, dir)
			if mntErr != nil {
				klog.Errorf("IsNotMountPoint check failed: %v", mntErr)
				return err
			}
			if !notMnt {
				// This is very odd, we don't expect it.  We'll try again next sync loop.
				klog.Errorf("%s is still mounted, despite call to unmount().  Will try again next sync loop.", dir)
				return err
			}
		}
		os.Remove(dir)
		return err
	}
	return nil
}


func (c *nfsUnmounter) TearDown() error

func (c *nfsUnmounter) TearDown() error {
	return c.TearDownAt(c.GetPath())
}


func (c *nfsUnmounter) TearDownAt(dir string) error {
	// Use extensiveMountPointCheck to consult /proc/mounts. We can't use faster
	// IsLikelyNotMountPoint (lstat()), since there may be root_squash on the
	// NFS server and kubelet may not be able to do lstat/stat() there.
	forceUnmounter, ok := c.mounter.(mount.MounterForceUnmounter)
	if ok {
		klog.V(4).Infof("Using force unmounter interface")
		return mount.CleanupMountWithForce(dir, forceUnmounter, true /* extensiveMountPointCheck */, unMountTimeout)
	}
	return mount.CleanupMountPoint(dir, c.mounter, true /* extensiveMountPointCheck */)
}


func getVolumeSource(spec *volume.Spec) (*v1.NFSVolumeSource, bool, error) {
	if spec.Volume != nil && spec.Volume.NFS != nil {
		return spec.Volume.NFS, spec.Volume.NFS.ReadOnly, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.NFS != nil {
		return spec.PersistentVolume.Spec.NFS, spec.ReadOnly, nil
	}

	return nil, false, fmt.Errorf("Spec does not reference a NFS volume type")
}


func getServerFromSource(source *v1.NFSVolumeSource) string {
	if netutil.IsIPv6String(source.Server) {
		return fmt.Sprintf("[%s]", source.Server)
	}
	return source.Server
}


func (n *noopExpandableVolumePluginInstance) ExpandVolumeDevice(spec *Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) {
	return newSize, nil
}


func (n *noopExpandableVolumePluginInstance) Init(host VolumeHost) error {
	return nil
}


func (n *noopExpandableVolumePluginInstance) GetPluginName() string {
	return n.spec.KubeletExpandablePluginName()
}


func (n *noopExpandableVolumePluginInstance) GetVolumeName(spec *Spec) (string, error) {
	return n.spec.Name(), nil
}


func (n *noopExpandableVolumePluginInstance) CanSupport(spec *Spec) bool {
	return true
}


func (n *noopExpandableVolumePluginInstance) RequiresRemount(spec *Spec) bool {
	return false
}


func (n *noopExpandableVolumePluginInstance) NewMounter(spec *Spec, podRef *v1.Pod, opts VolumeOptions) (Mounter, error) {
	return nil, nil
}


func (n *noopExpandableVolumePluginInstance) NewUnmounter(name string, podUID types.UID) (Unmounter, error) {
	return nil, nil
}


func (n *noopExpandableVolumePluginInstance) ConstructVolumeSpec(volumeName, mountPath string) (ReconstructedVolume, error) {
	return ReconstructedVolume{Spec: n.spec}, nil
}


func (n *noopExpandableVolumePluginInstance) SupportsMountOption() bool {
	return true
}


func (n *noopExpandableVolumePluginInstance) SupportsBulkVolumeVerification() bool {
	return false
}


func (n *noopExpandableVolumePluginInstance) RequiresFSResize() bool {
	return true
}


func (n *noopExpandableVolumePluginInstance) SupportsSELinuxContextMount(spec *Spec) (bool, error) {
	return false, nil
}


func (spec *Spec) Name() string

// Name returns the name of either Volume or PersistentVolume, one of which must not be nil.
func (spec *Spec) Name() string {
	switch {
	case spec.Volume != nil:
		return spec.Volume.Name
	case spec.PersistentVolume != nil:
		return spec.PersistentVolume.Name
	default:
		return ""
	}
}


// IsKubeletExpandable returns true for volume types that can be expanded only by the node
// and not the controller. Currently Flex volume is the only one in this category since
// it is typically not installed on the controller
func (spec *Spec) IsKubeletExpandable() bool {
	switch {
	case spec.Volume != nil:
		return spec.Volume.FlexVolume != nil
	case spec.PersistentVolume != nil:
		return spec.PersistentVolume.Spec.FlexVolume != nil
	default:
		return false
	}
}


// KubeletExpandablePluginName creates and returns a name for the plugin
// this is used in context on the controller where the plugin lookup fails
// as volume expansion on controller isn't supported, but a plugin name is
// required
func (spec *Spec) KubeletExpandablePluginName() string {
	switch {
	case spec.Volume != nil && spec.Volume.FlexVolume != nil:
		return spec.Volume.FlexVolume.Driver
	case spec.PersistentVolume != nil && spec.PersistentVolume.Spec.FlexVolume != nil:
		return spec.PersistentVolume.Spec.FlexVolume.Driver
	default:
		return ""
	}
}


// NewSpecFromVolume creates an Spec from an v1.Volume
func NewSpecFromVolume(vs *v1.Volume) *Spec {
	return &Spec{
		Volume: vs,
	}
}


// NewSpecFromPersistentVolume creates an Spec from an v1.PersistentVolume
func NewSpecFromPersistentVolume(pv *v1.PersistentVolume, readOnly bool) *Spec {
	return &Spec{
		PersistentVolume: pv,
		ReadOnly:         readOnly,
	}
}


// InitPlugins initializes each plugin.  All plugins must have unique names.
// This must be called exactly once before any New* methods are called on any
// plugins.
func (pm *VolumePluginMgr) InitPlugins(plugins []VolumePlugin, prober DynamicPluginProber, host VolumeHost) error {
	pm.mutex.Lock()
	defer pm.mutex.Unlock()

	pm.Host = host
	pm.loggedDeprecationWarnings = sets.NewString()

	if prober == nil {
		// Use a dummy prober to prevent nil deference.
		pm.prober = &dummyPluginProber{}
	} else {
		pm.prober = prober
	}
	if err := pm.prober.Init(); err != nil {
		// Prober init failure should not affect the initialization of other plugins.
		klog.ErrorS(err, "Error initializing dynamic plugin prober")
		pm.prober = &dummyPluginProber{}
	}

	if pm.plugins == nil {
		pm.plugins = map[string]VolumePlugin{}
	}
	if pm.probedPlugins == nil {
		pm.probedPlugins = map[string]VolumePlugin{}
	}

	allErrs := []error{}
	for _, plugin := range plugins {
		name := plugin.GetPluginName()
		if errs := validation.IsQualifiedName(name); len(errs) != 0 {
			allErrs = append(allErrs, fmt.Errorf("volume plugin has invalid name: %q: %s", name, strings.Join(errs, ";")))
			continue
		}

		if _, found := pm.plugins[name]; found {
			allErrs = append(allErrs, fmt.Errorf("volume plugin %q was registered more than once", name))
			continue
		}
		err := plugin.Init(host)
		if err != nil {
			klog.ErrorS(err, "Failed to load volume plugin", "pluginName", name)
			allErrs = append(allErrs, err)
			continue
		}
		pm.plugins[name] = plugin
		klog.V(1).InfoS("Loaded volume plugin", "pluginName", name)
	}
	return utilerrors.NewAggregate(allErrs)
}


func (pm *VolumePluginMgr) initProbedPlugin(probedPlugin VolumePlugin) error {
	name := probedPlugin.GetPluginName()
	if errs := validation.IsQualifiedName(name); len(errs) != 0 {
		return fmt.Errorf("volume plugin has invalid name: %q: %s", name, strings.Join(errs, ";"))
	}

	err := probedPlugin.Init(pm.Host)
	if err != nil {
		return fmt.Errorf("failed to load volume plugin %s, error: %s", name, err.Error())
	}

	klog.V(1).InfoS("Loaded volume plugin", "pluginName", name)
	return nil
}


// FindPluginBySpec looks for a plugin that can support a given volume
// specification.  If no plugins can support or more than one plugin can
// support it, return error.
func (pm *VolumePluginMgr) FindPluginBySpec(spec *Spec) (VolumePlugin, error) {
	pm.mutex.RLock()
	defer pm.mutex.RUnlock()

	if spec == nil {
		return nil, fmt.Errorf("could not find plugin because volume spec is nil")
	}

	var match VolumePlugin
	matchedPluginNames := []string{}
	for _, v := range pm.plugins {
		if v.CanSupport(spec) {
			match = v
			matchedPluginNames = append(matchedPluginNames, v.GetPluginName())
		}
	}

	pm.refreshProbedPlugins()
	for _, plugin := range pm.probedPlugins {
		if plugin.CanSupport(spec) {
			match = plugin
			matchedPluginNames = append(matchedPluginNames, plugin.GetPluginName())
		}
	}

	if len(matchedPluginNames) == 0 {
		return nil, fmt.Errorf("no volume plugin matched")
	}
	if len(matchedPluginNames) > 1 {
		return nil, fmt.Errorf("multiple volume plugins matched: %s", strings.Join(matchedPluginNames, ","))
	}

	return match, nil
}


// FindPluginByName fetches a plugin by name or by legacy name.  If no plugin
// is found, returns error.
func (pm *VolumePluginMgr) FindPluginByName(name string) (VolumePlugin, error) {
	pm.mutex.RLock()
	defer pm.mutex.RUnlock()

	// Once we can get rid of legacy names we can reduce this to a map lookup.
	var match VolumePlugin
	if v, found := pm.plugins[name]; found {
		match = v
	}

	pm.refreshProbedPlugins()
	if plugin, found := pm.probedPlugins[name]; found {
		if match != nil {
			return nil, fmt.Errorf("multiple volume plugins matched: %s and %s", match.GetPluginName(), plugin.GetPluginName())
		}
		match = plugin
	}

	if match == nil {
		return nil, fmt.Errorf("no volume plugin matched name: %s", name)
	}
	return match, nil
}


// Check if probedPlugin cache update is required.
// If it is, initialize all probed plugins and replace the cache with them.
func (pm *VolumePluginMgr) refreshProbedPlugins() {
	events, err := pm.prober.Probe()

	if err != nil {
		klog.ErrorS(err, "Error dynamically probing plugins")
	}

	// because the probe function can return a list of valid plugins
	// even when an error is present we still must add the plugins
	// or they will be skipped because each event only fires once
	for _, event := range events {
		if event.Op == ProbeAddOrUpdate {
			if err := pm.initProbedPlugin(event.Plugin); err != nil {
				klog.ErrorS(err, "Error initializing dynamically probed plugin",
					"pluginName", event.Plugin.GetPluginName())
				continue
			}
			pm.probedPlugins[event.Plugin.GetPluginName()] = event.Plugin
		} else if event.Op == ProbeRemove {
			// Plugin is not available on ProbeRemove event, only PluginName
			delete(pm.probedPlugins, event.PluginName)
		} else {
			klog.ErrorS(nil, "Unknown Operation on PluginName.",
				"pluginName", event.Plugin.GetPluginName())
		}
	}
}


// ListVolumePluginWithLimits returns plugins that have volume limits on nodes
func (pm *VolumePluginMgr) ListVolumePluginWithLimits() []VolumePluginWithAttachLimits {
	pm.mutex.RLock()
	defer pm.mutex.RUnlock()

	matchedPlugins := []VolumePluginWithAttachLimits{}
	for _, v := range pm.plugins {
		if plugin, ok := v.(VolumePluginWithAttachLimits); ok {
			matchedPlugins = append(matchedPlugins, plugin)
		}
	}
	return matchedPlugins
}


// FindPersistentPluginBySpec looks for a persistent volume plugin that can
// support a given volume specification.  If no plugin is found, return an
// error
func (pm *VolumePluginMgr) FindPersistentPluginBySpec(spec *Spec) (PersistentVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		return nil, fmt.Errorf("could not find volume plugin for spec: %#v", spec)
	}
	if persistentVolumePlugin, ok := volumePlugin.(PersistentVolumePlugin); ok {
		return persistentVolumePlugin, nil
	}
	return nil, fmt.Errorf("no persistent volume plugin matched")
}


// FindVolumePluginWithLimitsBySpec returns volume plugin that has a limit on how many
// of them can be attached to a node
func (pm *VolumePluginMgr) FindVolumePluginWithLimitsBySpec(spec *Spec) (VolumePluginWithAttachLimits, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		return nil, fmt.Errorf("could not find volume plugin for spec : %#v", spec)
	}

	if limitedPlugin, ok := volumePlugin.(VolumePluginWithAttachLimits); ok {
		return limitedPlugin, nil
	}
	return nil, fmt.Errorf("no plugin with limits found")
}


// FindPersistentPluginByName fetches a persistent volume plugin by name.  If
// no plugin is found, returns error.
func (pm *VolumePluginMgr) FindPersistentPluginByName(name string) (PersistentVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginByName(name)
	if err != nil {
		return nil, err
	}
	if persistentVolumePlugin, ok := volumePlugin.(PersistentVolumePlugin); ok {
		return persistentVolumePlugin, nil
	}
	return nil, fmt.Errorf("no persistent volume plugin matched")
}


// FindRecyclablePluginByName fetches a persistent volume plugin by name.  If
// no plugin is found, returns error.
func (pm *VolumePluginMgr) FindRecyclablePluginBySpec(spec *Spec) (RecyclableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		return nil, err
	}
	if recyclableVolumePlugin, ok := volumePlugin.(RecyclableVolumePlugin); ok {
		return recyclableVolumePlugin, nil
	}
	return nil, fmt.Errorf("no recyclable volume plugin matched")
}


// FindProvisionablePluginByName fetches  a persistent volume plugin by name.  If
// no plugin is found, returns error.
func (pm *VolumePluginMgr) FindProvisionablePluginByName(name string) (ProvisionableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginByName(name)
	if err != nil {
		return nil, err
	}
	if provisionableVolumePlugin, ok := volumePlugin.(ProvisionableVolumePlugin); ok {
		return provisionableVolumePlugin, nil
	}
	return nil, fmt.Errorf("no provisionable volume plugin matched")
}


// FindDeletablePluginBySpec fetches a persistent volume plugin by spec.  If
// no plugin is found, returns error.
func (pm *VolumePluginMgr) FindDeletablePluginBySpec(spec *Spec) (DeletableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		return nil, err
	}
	if deletableVolumePlugin, ok := volumePlugin.(DeletableVolumePlugin); ok {
		return deletableVolumePlugin, nil
	}
	return nil, fmt.Errorf("no deletable volume plugin matched")
}


// FindDeletablePluginByName fetches a persistent volume plugin by name.  If
// no plugin is found, returns error.
func (pm *VolumePluginMgr) FindDeletablePluginByName(name string) (DeletableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginByName(name)
	if err != nil {
		return nil, err
	}
	if deletableVolumePlugin, ok := volumePlugin.(DeletableVolumePlugin); ok {
		return deletableVolumePlugin, nil
	}
	return nil, fmt.Errorf("no deletable volume plugin matched")
}


// FindCreatablePluginBySpec fetches a persistent volume plugin by name.  If
// no plugin is found, returns error.
func (pm *VolumePluginMgr) FindCreatablePluginBySpec(spec *Spec) (ProvisionableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		return nil, err
	}
	if provisionableVolumePlugin, ok := volumePlugin.(ProvisionableVolumePlugin); ok {
		return provisionableVolumePlugin, nil
	}
	return nil, fmt.Errorf("no creatable volume plugin matched")
}


// FindAttachablePluginBySpec fetches a persistent volume plugin by spec.
// Unlike the other "FindPlugin" methods, this does not return error if no
// plugin is found.  All volumes require a mounter and unmounter, but not
// every volume will have an attacher/detacher.
func (pm *VolumePluginMgr) FindAttachablePluginBySpec(spec *Spec) (AttachableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		return nil, err
	}
	if attachableVolumePlugin, ok := volumePlugin.(AttachableVolumePlugin); ok {
		if canAttach, err := attachableVolumePlugin.CanAttach(spec); err != nil {
			return nil, err
		} else if canAttach {
			return attachableVolumePlugin, nil
		}
	}
	return nil, nil
}


// FindAttachablePluginByName fetches an attachable volume plugin by name.
// Unlike the other "FindPlugin" methods, this does not return error if no
// plugin is found.  All volumes require a mounter and unmounter, but not
// every volume will have an attacher/detacher.
func (pm *VolumePluginMgr) FindAttachablePluginByName(name string) (AttachableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginByName(name)
	if err != nil {
		return nil, err
	}
	if attachablePlugin, ok := volumePlugin.(AttachableVolumePlugin); ok {
		return attachablePlugin, nil
	}
	return nil, nil
}


// FindDeviceMountablePluginBySpec fetches a persistent volume plugin by spec.
func (pm *VolumePluginMgr) FindDeviceMountablePluginBySpec(spec *Spec) (DeviceMountableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		return nil, err
	}
	if deviceMountableVolumePlugin, ok := volumePlugin.(DeviceMountableVolumePlugin); ok {
		if canMount, err := deviceMountableVolumePlugin.CanDeviceMount(spec); err != nil {
			return nil, err
		} else if canMount {
			return deviceMountableVolumePlugin, nil
		}
	}
	return nil, nil
}


// FindDeviceMountablePluginByName fetches a devicemountable volume plugin by name.
func (pm *VolumePluginMgr) FindDeviceMountablePluginByName(name string) (DeviceMountableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginByName(name)
	if err != nil {
		return nil, err
	}
	if deviceMountableVolumePlugin, ok := volumePlugin.(DeviceMountableVolumePlugin); ok {
		return deviceMountableVolumePlugin, nil
	}
	return nil, nil
}


// FindExpandablePluginBySpec fetches a persistent volume plugin by spec.
func (pm *VolumePluginMgr) FindExpandablePluginBySpec(spec *Spec) (ExpandableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		if spec.IsKubeletExpandable() {
			// for kubelet expandable volumes, return a noop plugin that
			// returns success for expand on the controller
			klog.V(4).InfoS("FindExpandablePluginBySpec -> returning noopExpandableVolumePluginInstance", "specName", spec.Name())
			return &noopExpandableVolumePluginInstance{spec}, nil
		}
		klog.V(4).InfoS("FindExpandablePluginBySpec -> err", "specName", spec.Name(), "err", err)
		return nil, err
	}

	if expandableVolumePlugin, ok := volumePlugin.(ExpandableVolumePlugin); ok {
		return expandableVolumePlugin, nil
	}
	return nil, nil
}


// FindExpandablePluginBySpec fetches a persistent volume plugin by name.
func (pm *VolumePluginMgr) FindExpandablePluginByName(name string) (ExpandableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginByName(name)
	if err != nil {
		return nil, err
	}

	if expandableVolumePlugin, ok := volumePlugin.(ExpandableVolumePlugin); ok {
		return expandableVolumePlugin, nil
	}
	return nil, nil
}


// FindMapperPluginBySpec fetches a block volume plugin by spec.
func (pm *VolumePluginMgr) FindMapperPluginBySpec(spec *Spec) (BlockVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		return nil, err
	}

	if blockVolumePlugin, ok := volumePlugin.(BlockVolumePlugin); ok {
		return blockVolumePlugin, nil
	}
	return nil, nil
}


// FindMapperPluginByName fetches a block volume plugin by name.
func (pm *VolumePluginMgr) FindMapperPluginByName(name string) (BlockVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginByName(name)
	if err != nil {
		return nil, err
	}

	if blockVolumePlugin, ok := volumePlugin.(BlockVolumePlugin); ok {
		return blockVolumePlugin, nil
	}
	return nil, nil
}


// FindNodeExpandablePluginBySpec fetches a persistent volume plugin by spec
func (pm *VolumePluginMgr) FindNodeExpandablePluginBySpec(spec *Spec) (NodeExpandableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		return nil, err
	}
	if fsResizablePlugin, ok := volumePlugin.(NodeExpandableVolumePlugin); ok {
		return fsResizablePlugin, nil
	}
	return nil, nil
}


// FindNodeExpandablePluginByName fetches a persistent volume plugin by name
func (pm *VolumePluginMgr) FindNodeExpandablePluginByName(name string) (NodeExpandableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginByName(name)
	if err != nil {
		return nil, err
	}

	if fsResizablePlugin, ok := volumePlugin.(NodeExpandableVolumePlugin); ok {
		return fsResizablePlugin, nil
	}

	return nil, nil
}


func (pm *VolumePluginMgr) Run(stopCh <-chan struct{}) {
	kletHost, ok := pm.Host.(KubeletVolumeHost)
	if ok {
		// start informer for CSIDriver
		informerFactory := kletHost.GetInformerFactory()
		informerFactory.Start(stopCh)
		informerFactory.WaitForCacheSync(stopCh)
	}
}


// NewPersistentVolumeRecyclerPodTemplate creates a template for a recycler
// pod.  By default, a recycler pod simply runs "rm -rf" on a volume and tests
// for emptiness.  Most attributes of the template will be correct for most
// plugin implementations.  The following attributes can be overridden per
// plugin via configuration:
//
//  1. pod.Spec.Volumes[0].VolumeSource must be overridden.  Recycler
//     implementations without a valid VolumeSource will fail.
//  2. pod.GenerateName helps distinguish recycler pods by name.  Recommended.
//     Default is "pv-recycler-".
//  3. pod.Spec.ActiveDeadlineSeconds gives the recycler pod a maximum timeout
//     before failing.  Recommended.  Default is 60 seconds.
//
// See HostPath and NFS for working recycler examples
func NewPersistentVolumeRecyclerPodTemplate() *v1.Pod {
	timeout := int64(60)
	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "pv-recycler-",
			Namespace:    metav1.NamespaceDefault,
		},
		Spec: v1.PodSpec{
			ActiveDeadlineSeconds: &timeout,
			RestartPolicy:         v1.RestartPolicyNever,
			Volumes: []v1.Volume{
				{
					Name: "vol",
					// IMPORTANT!  All plugins using this template MUST
					// override pod.Spec.Volumes[0].VolumeSource Recycler
					// implementations without a valid VolumeSource will fail.
					VolumeSource: v1.VolumeSource{},
				},
			},
			Containers: []v1.Container{
				{
					Name:    "pv-recycler",
					Image:   "registry.k8s.io/debian-base:v2.0.0",
					Command: []string{"/bin/sh"},
					Args:    []string{"-c", "test -e /scrub && rm -rf /scrub/..?* /scrub/.[!.]* /scrub/*  && test -z \"$(ls -A /scrub)\" || exit 1"},
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      "vol",
							MountPath: "/scrub",
						},
					},
				},
			},
		},
	}
	return pod
}


// Check validity of recycle pod template
// List of checks:
// - at least one volume is defined in the recycle pod template
// If successful, returns nil
// if unsuccessful, returns an error.
func ValidateRecyclerPodTemplate(pod *v1.Pod) error {
	if len(pod.Spec.Volumes) < 1 {
		return fmt.Errorf("does not contain any volume(s)")
	}
	return nil
}


func (*dummyPluginProber) Init() error                  { return nil }


func (*dummyPluginProber) Probe() ([]ProbeEvent, error) { return nil, nil }


// ProbeVolumePlugins is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&portworxVolumePlugin{nil, nil}}
}


func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
	return host.GetPodVolumeDir(uid, utilstrings.EscapeQualifiedName(portworxVolumePluginName), volName)
}


func (plugin *portworxVolumePlugin) IsMigratedToCSI() bool {
	return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationPortworx)
}


func (plugin *portworxVolumePlugin) Init(host volume.VolumeHost) error {
	client, err := volumeclient.NewDriverClient(
		fmt.Sprintf("http://%s", net.JoinHostPort(host.GetHostName(), strconv.Itoa(osdMgmtDefaultPort))),
		pxdDriverName, osdDriverVersion, pxDriverName)
	if err != nil {
		return err
	}

	plugin.host = host
	plugin.util = &portworxVolumeUtil{
		portworxClient: client,
	}

	return nil
}


func (plugin *portworxVolumePlugin) GetPluginName() string {
	return portworxVolumePluginName
}


func (plugin *portworxVolumePlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return volumeSource.VolumeID, nil
}


func (plugin *portworxVolumePlugin) CanSupport(spec *volume.Spec) bool {
	return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.PortworxVolume != nil) ||
		(spec.Volume != nil && spec.Volume.PortworxVolume != nil)
}


func (plugin *portworxVolumePlugin) RequiresRemount(spec *volume.Spec) bool {
	return false
}


func (plugin *portworxVolumePlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
	return []v1.PersistentVolumeAccessMode{
		v1.ReadWriteOnce,
		v1.ReadWriteMany,
	}
}


func (plugin *portworxVolumePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
	return plugin.newMounterInternal(spec, pod.UID, plugin.util, plugin.host.GetMounter(plugin.GetPluginName()))
}


func (plugin *portworxVolumePlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager portworxManager, mounter mount.Interface) (volume.Mounter, error) {
	pwx, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}

	volumeID := pwx.VolumeID
	fsType := pwx.FSType

	return &portworxVolumeMounter{
		portworxVolume: &portworxVolume{
			podUID:          podUID,
			volName:         spec.Name(),
			volumeID:        volumeID,
			manager:         manager,
			mounter:         mounter,
			plugin:          plugin,
			MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, spec.Name(), plugin.host)),
		},
		fsType:      fsType,
		readOnly:    readOnly,
		diskMounter: util.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host)}, nil
}


func (plugin *portworxVolumePlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	return plugin.newUnmounterInternal(volName, podUID, plugin.util, plugin.host.GetMounter(plugin.GetPluginName()))
}


func (plugin *portworxVolumePlugin) newUnmounterInternal(volName string, podUID types.UID, manager portworxManager,
	mounter mount.Interface) (volume.Unmounter, error) {
	return &portworxVolumeUnmounter{
		&portworxVolume{
			podUID:          podUID,
			volName:         volName,
			manager:         manager,
			mounter:         mounter,
			plugin:          plugin,
			MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)),
		}}, nil
}


func (plugin *portworxVolumePlugin) NewDeleter(logger klog.Logger, spec *volume.Spec) (volume.Deleter, error) {
	return plugin.newDeleterInternal(spec, plugin.util)
}


func (plugin *portworxVolumePlugin) newDeleterInternal(spec *volume.Spec, manager portworxManager) (volume.Deleter, error) {
	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.PortworxVolume == nil {
		return nil, fmt.Errorf("spec.PersistentVolumeSource.PortworxVolume is nil")
	}

	return &portworxVolumeDeleter{
		portworxVolume: &portworxVolume{
			volName:  spec.Name(),
			volumeID: spec.PersistentVolume.Spec.PortworxVolume.VolumeID,
			manager:  manager,
			plugin:   plugin,
		}}, nil
}


func (plugin *portworxVolumePlugin) NewProvisioner(logger klog.Logger, options volume.VolumeOptions) (volume.Provisioner, error) {
	return plugin.newProvisionerInternal(options, plugin.util)
}


func (plugin *portworxVolumePlugin) newProvisionerInternal(options volume.VolumeOptions, manager portworxManager) (volume.Provisioner, error) {
	return &portworxVolumeProvisioner{
		portworxVolume: &portworxVolume{
			manager: manager,
			plugin:  plugin,
		},
		options: options,
	}, nil
}


func (plugin *portworxVolumePlugin) RequiresFSResize() bool {
	return false
}


func (plugin *portworxVolumePlugin) ExpandVolumeDevice(
	spec *volume.Spec,
	newSize resource.Quantity,
	oldSize resource.Quantity) (resource.Quantity, error) {
	klog.V(4).Infof("Expanding: %s from %v to %v", spec.Name(), oldSize, newSize)
	err := plugin.util.ResizeVolume(spec, newSize, plugin.host)
	if err != nil {
		return oldSize, err
	}

	klog.V(4).Infof("Successfully resized %s to %v", spec.Name(), newSize)
	return newSize, nil
}


func (plugin *portworxVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
	portworxVolume := &v1.Volume{
		Name: volumeName,
		VolumeSource: v1.VolumeSource{
			PortworxVolume: &v1.PortworxVolumeSource{
				VolumeID: volumeName,
			},
		},
	}
	return volume.ReconstructedVolume{
		Spec: volume.NewSpecFromVolume(portworxVolume),
	}, nil
}


func (plugin *portworxVolumePlugin) SupportsMountOption() bool {
	return false
}


func (plugin *portworxVolumePlugin) SupportsBulkVolumeVerification() bool {
	return false
}


func (plugin *portworxVolumePlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return false, nil
}


func getVolumeSource(

func getVolumeSource(
	spec *volume.Spec) (*v1.PortworxVolumeSource, bool, error) {
	if spec.Volume != nil && spec.Volume.PortworxVolume != nil {
		return spec.Volume.PortworxVolume, spec.Volume.PortworxVolume.ReadOnly, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.PortworxVolume != nil {
		return spec.PersistentVolume.Spec.PortworxVolume, spec.ReadOnly, nil
	}

	return nil, false, fmt.Errorf("Spec does not reference a Portworx Volume type")
}


func (b *portworxVolumeMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:       b.readOnly,
		Managed:        !b.readOnly,
		SELinuxRelabel: false,
	}
}


// SetUp attaches the disk and bind mounts to the volume path.
func (b *portworxVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error {
	return b.SetUpAt(b.GetPath(), mounterArgs)
}