// MakePayload function is exported so that it can be called from the projection volume driver
func MakePayload(mappings []v1.KeyToPath, secret *v1.Secret, defaultMode *int32, optional bool) (map[string]volumeutil.FileProjection, error) {
	if defaultMode == nil {
		return nil, fmt.Errorf("no defaultMode used, not even the default value for it")
	}

	payload := make(map[string]volumeutil.FileProjection, len(secret.Data))
	var fileProjection volumeutil.FileProjection

	if len(mappings) == 0 {
		for name, data := range secret.Data {
			fileProjection.Data = []byte(data)
			fileProjection.Mode = *defaultMode
			payload[name] = fileProjection
		}
	} else {
		for _, ktp := range mappings {
			content, ok := secret.Data[ktp.Key]
			if !ok {
				if optional {
					continue
				}
				errMsg := fmt.Sprintf("references non-existent secret key: %s", ktp.Key)
				klog.Errorf(errMsg)
				return nil, fmt.Errorf(errMsg)
			}

			fileProjection.Data = []byte(content)
			if ktp.Mode != nil {
				fileProjection.Mode = *ktp.Mode
			} else {
				fileProjection.Mode = *defaultMode
			}
			payload[ktp.Path] = fileProjection
		}
	}
	return payload, nil
}


func totalSecretBytes(secret *v1.Secret) int {
	totalSize := 0
	for _, bytes := range secret.Data {
		totalSize += len(bytes)
	}

	return totalSize
}


func (c *secretVolumeUnmounter) TearDown() error {
	return c.TearDownAt(c.GetPath())
}


func (c *secretVolumeUnmounter) TearDownAt(dir string) error {
	return volumeutil.UnmountViaEmptyDir(dir, c.plugin.host, c.volName, wrappedVolumeSpec(), c.podUID)
}


func getVolumeSource(spec *volume.Spec) (*v1.SecretVolumeSource, bool) {
	var readOnly bool
	var volumeSource *v1.SecretVolumeSource

	if spec.Volume != nil && spec.Volume.Secret != nil {
		volumeSource = spec.Volume.Secret
		readOnly = spec.ReadOnly
	}

	return volumeSource, readOnly
}


// NewAtomicWriter creates a new AtomicWriter configured to write to the given
// target directory, or returns an error if the target directory does not exist.
func NewAtomicWriter(targetDir string, logContext string) (*AtomicWriter, error) {
	_, err := os.Stat(targetDir)
	if os.IsNotExist(err) {
		return nil, err
	}

	return &AtomicWriter{targetDir: targetDir, logContext: logContext}, nil
}


// Write does an atomic projection of the given payload into the writer's target
// directory.  Input paths must not begin with '..'.
// setPerms is an optional pointer to a function that caller can provide to set the
// permissions of the newly created files before they are published. The function is
// passed subPath which is the name of the timestamped directory that was created
// under target directory.
//
// The Write algorithm is:
//
//  1. The payload is validated; if the payload is invalid, the function returns
//
//  2. The current timestamped directory is detected by reading the data directory
//     symlink
//
//  3. The old version of the volume is walked to determine whether any
//     portion of the payload was deleted and is still present on disk.
//
//  4. The data in the current timestamped directory is compared to the projected
//     data to determine if an update is required.
//
//  5. A new timestamped dir is created.
//
//  6. The payload is written to the new timestamped directory.
//
//  7. Permissions are set (if setPerms is not nil) on the new timestamped directory and files.
//
//  8. A symlink to the new timestamped directory ..data_tmp is created that will
//     become the new data directory.
//
//  9. The new data directory symlink is renamed to the data directory; rename is atomic.
//
//  10. Symlinks and directory for new user-visible files are created (if needed).
//
//     For example, consider the files:
//     <target-dir>/podName
//     <target-dir>/user/labels
//     <target-dir>/k8s/annotations
//
//     The user visible files are symbolic links into the internal data directory:
//     <target-dir>/podName         -> ..data/podName
//     <target-dir>/usr -> ..data/usr
//     <target-dir>/k8s -> ..data/k8s
//
//     The data directory itself is a link to a timestamped directory with
//     the real data:
//     <target-dir>/..data          -> ..2016_02_01_15_04_05.12345678/
//     NOTE(claudiub): We need to create these symlinks AFTER we've finished creating and
//     linking everything else. On Windows, if a target does not exist, the created symlink
//     will not work properly if the target ends up being a directory.
//
//  11. Old paths are removed from the user-visible portion of the target directory.
//
//  12. The previous timestamped directory is removed, if it exists.
func (w *AtomicWriter) Write(payload map[string]FileProjection, setPerms func(subPath string) error) error {
	// (1)
	cleanPayload, err := validatePayload(payload)
	if err != nil {
		klog.Errorf("%s: invalid payload: %v", w.logContext, err)
		return err
	}

	// (2)
	dataDirPath := filepath.Join(w.targetDir, dataDirName)
	oldTsDir, err := os.Readlink(dataDirPath)
	if err != nil {
		if !os.IsNotExist(err) {
			klog.Errorf("%s: error reading link for data directory: %v", w.logContext, err)
			return err
		}
		// although Readlink() returns "" on err, don't be fragile by relying on it (since it's not specified in docs)
		// empty oldTsDir indicates that it didn't exist
		oldTsDir = ""
	}
	oldTsPath := filepath.Join(w.targetDir, oldTsDir)

	var pathsToRemove sets.String
	// if there was no old version, there's nothing to remove
	if len(oldTsDir) != 0 {
		// (3)
		pathsToRemove, err = w.pathsToRemove(cleanPayload, oldTsPath)
		if err != nil {
			klog.Errorf("%s: error determining user-visible files to remove: %v", w.logContext, err)
			return err
		}

		// (4)
		if should, err := shouldWritePayload(cleanPayload, oldTsPath); err != nil {
			klog.Errorf("%s: error determining whether payload should be written to disk: %v", w.logContext, err)
			return err
		} else if !should && len(pathsToRemove) == 0 {
			klog.V(4).Infof("%s: no update required for target directory %v", w.logContext, w.targetDir)
			return nil
		} else {
			klog.V(4).Infof("%s: write required for target directory %v", w.logContext, w.targetDir)
		}
	}

	// (5)
	tsDir, err := w.newTimestampDir()
	if err != nil {
		klog.V(4).Infof("%s: error creating new ts data directory: %v", w.logContext, err)
		return err
	}
	tsDirName := filepath.Base(tsDir)

	// (6)
	if err = w.writePayloadToDir(cleanPayload, tsDir); err != nil {
		klog.Errorf("%s: error writing payload to ts data directory %s: %v", w.logContext, tsDir, err)
		return err
	}
	klog.V(4).Infof("%s: performed write of new data to ts data directory: %s", w.logContext, tsDir)

	// (7)
	if setPerms != nil {
		if err := setPerms(tsDirName); err != nil {
			klog.Errorf("%s: error applying ownership settings: %v", w.logContext, err)
			return err
		}
	}

	// (8)
	newDataDirPath := filepath.Join(w.targetDir, newDataDirName)
	if err = os.Symlink(tsDirName, newDataDirPath); err != nil {
		os.RemoveAll(tsDir)
		klog.Errorf("%s: error creating symbolic link for atomic update: %v", w.logContext, err)
		return err
	}

	// (9)
	if runtime.GOOS == "windows" {
		os.Remove(dataDirPath)
		err = os.Symlink(tsDirName, dataDirPath)
		os.Remove(newDataDirPath)
	} else {
		err = os.Rename(newDataDirPath, dataDirPath)
	}
	if err != nil {
		os.Remove(newDataDirPath)
		os.RemoveAll(tsDir)
		klog.Errorf("%s: error renaming symbolic link for data directory %s: %v", w.logContext, newDataDirPath, err)
		return err
	}

	// (10)
	if err = w.createUserVisibleFiles(cleanPayload); err != nil {
		klog.Errorf("%s: error creating visible symlinks in %s: %v", w.logContext, w.targetDir, err)
		return err
	}

	// (11)
	if err = w.removeUserVisiblePaths(pathsToRemove); err != nil {
		klog.Errorf("%s: error removing old visible symlinks: %v", w.logContext, err)
		return err
	}

	// (12)
	if len(oldTsDir) > 0 {
		if err = os.RemoveAll(oldTsPath); err != nil {
			klog.Errorf("%s: error removing old data directory %s: %v", w.logContext, oldTsDir, err)
			return err
		}
	}

	return nil
}


// validatePayload returns an error if any path in the payload returns a copy of the payload with the paths cleaned.
func validatePayload(payload map[string]FileProjection) (map[string]FileProjection, error) {
	cleanPayload := make(map[string]FileProjection)
	for k, content := range payload {
		if err := validatePath(k); err != nil {
			return nil, err
		}

		cleanPayload[filepath.Clean(k)] = content
	}

	return cleanPayload, nil
}


// validatePath validates a single path, returning an error if the path is
// invalid.  paths may not:
//
// 1. be absolute
// 2. contain '..' as an element
// 3. start with '..'
// 4. contain filenames larger than 255 characters
// 5. be longer than 4096 characters
func validatePath(targetPath string) error {
	// TODO: somehow unify this with the similar api validation,
	// validateVolumeSourcePath; the error semantics are just different enough
	// from this that it was time-prohibitive trying to find the right
	// refactoring to re-use.
	if targetPath == "" {
		return fmt.Errorf("invalid path: must not be empty: %q", targetPath)
	}
	if path.IsAbs(targetPath) {
		return fmt.Errorf("invalid path: must be relative path: %s", targetPath)
	}

	if len(targetPath) > maxPathLength {
		return fmt.Errorf("invalid path: must be less than or equal to %d characters", maxPathLength)
	}

	items := strings.Split(targetPath, string(os.PathSeparator))
	for _, item := range items {
		if item == ".." {
			return fmt.Errorf("invalid path: must not contain '..': %s", targetPath)
		}
		if len(item) > maxFileNameLength {
			return fmt.Errorf("invalid path: filenames must be less than or equal to %d characters", maxFileNameLength)
		}
	}
	if strings.HasPrefix(items[0], "..") && len(items[0]) > 2 {
		return fmt.Errorf("invalid path: must not start with '..': %s", targetPath)
	}

	return nil
}


// shouldWritePayload returns whether the payload should be written to disk.
func shouldWritePayload(payload map[string]FileProjection, oldTsDir string) (bool, error) {
	for userVisiblePath, fileProjection := range payload {
		shouldWrite, err := shouldWriteFile(filepath.Join(oldTsDir, userVisiblePath), fileProjection.Data)
		if err != nil {
			return false, err
		}

		if shouldWrite {
			return true, nil
		}
	}

	return false, nil
}


// shouldWriteFile returns whether a new version of a file should be written to disk.
func shouldWriteFile(path string, content []byte) (bool, error) {
	_, err := os.Lstat(path)
	if os.IsNotExist(err) {
		return true, nil
	}

	contentOnFs, err := ioutil.ReadFile(path)
	if err != nil {
		return false, err
	}

	return !bytes.Equal(content, contentOnFs), nil
}


// pathsToRemove walks the current version of the data directory and
// determines which paths should be removed (if any) after the payload is
// written to the target directory.
func (w *AtomicWriter) pathsToRemove(payload map[string]FileProjection, oldTsDir string) (sets.String, error) {
	paths := sets.NewString()
	visitor := func(path string, info os.FileInfo, err error) error {
		relativePath := strings.TrimPrefix(path, oldTsDir)
		relativePath = strings.TrimPrefix(relativePath, string(os.PathSeparator))
		if relativePath == "" {
			return nil
		}

		paths.Insert(relativePath)
		return nil
	}

	err := filepath.Walk(oldTsDir, visitor)
	if os.IsNotExist(err) {
		return nil, nil
	} else if err != nil {
		return nil, err
	}
	klog.V(5).Infof("%s: current paths:   %+v", w.targetDir, paths.List())

	newPaths := sets.NewString()
	for file := range payload {
		// add all subpaths for the payload to the set of new paths
		// to avoid attempting to remove non-empty dirs
		for subPath := file; subPath != ""; {
			newPaths.Insert(subPath)
			subPath, _ = filepath.Split(subPath)
			subPath = strings.TrimSuffix(subPath, string(os.PathSeparator))
		}
	}
	klog.V(5).Infof("%s: new paths:       %+v", w.targetDir, newPaths.List())

	result := paths.Difference(newPaths)
	klog.V(5).Infof("%s: paths to remove: %+v", w.targetDir, result)

	return result, nil
}


// newTimestampDir creates a new timestamp directory
func (w *AtomicWriter) newTimestampDir() (string, error) {
	tsDir, err := ioutil.TempDir(w.targetDir, time.Now().UTC().Format("..2006_01_02_15_04_05."))
	if err != nil {
		klog.Errorf("%s: unable to create new temp directory: %v", w.logContext, err)
		return "", err
	}

	// 0755 permissions are needed to allow 'group' and 'other' to recurse the
	// directory tree.  do a chmod here to ensure that permissions are set correctly
	// regardless of the process' umask.
	err = os.Chmod(tsDir, 0755)
	if err != nil {
		klog.Errorf("%s: unable to set mode on new temp directory: %v", w.logContext, err)
		return "", err
	}

	return tsDir, nil
}


// writePayloadToDir writes the given payload to the given directory.  The
// directory must exist.
func (w *AtomicWriter) writePayloadToDir(payload map[string]FileProjection, dir string) error {
	for userVisiblePath, fileProjection := range payload {
		content := fileProjection.Data
		mode := os.FileMode(fileProjection.Mode)
		fullPath := filepath.Join(dir, userVisiblePath)
		baseDir, _ := filepath.Split(fullPath)

		if err := os.MkdirAll(baseDir, os.ModePerm); err != nil {
			klog.Errorf("%s: unable to create directory %s: %v", w.logContext, baseDir, err)
			return err
		}

		if err := ioutil.WriteFile(fullPath, content, mode); err != nil {
			klog.Errorf("%s: unable to write file %s with mode %v: %v", w.logContext, fullPath, mode, err)
			return err
		}
		// Chmod is needed because ioutil.WriteFile() ends up calling
		// open(2) to create the file, so the final mode used is "mode &
		// ~umask". But we want to make sure the specified mode is used
		// in the file no matter what the umask is.
		if err := os.Chmod(fullPath, mode); err != nil {
			klog.Errorf("%s: unable to change file %s with mode %v: %v", w.logContext, fullPath, mode, err)
			return err
		}

		if fileProjection.FsUser == nil {
			continue
		}
		if err := os.Chown(fullPath, int(*fileProjection.FsUser), -1); err != nil {
			klog.Errorf("%s: unable to change file %s with owner %v: %v", w.logContext, fullPath, int(*fileProjection.FsUser), err)
			return err
		}
	}

	return nil
}


// createUserVisibleFiles creates the relative symlinks for all the
// files configured in the payload. If the directory in a file path does not
// exist, it is created.
//
// Viz:
// For files: "bar", "foo/bar", "baz/bar", "foo/baz/blah"
// the following symlinks are created:
// bar -> ..data/bar
// foo -> ..data/foo
// baz -> ..data/baz
func (w *AtomicWriter) createUserVisibleFiles(payload map[string]FileProjection) error {
	for userVisiblePath := range payload {
		slashpos := strings.Index(userVisiblePath, string(os.PathSeparator))
		if slashpos == -1 {
			slashpos = len(userVisiblePath)
		}
		linkname := userVisiblePath[:slashpos]
		_, err := os.Readlink(filepath.Join(w.targetDir, linkname))
		if err != nil && os.IsNotExist(err) {
			// The link into the data directory for this path doesn't exist; create it
			visibleFile := filepath.Join(w.targetDir, linkname)
			dataDirFile := filepath.Join(dataDirName, linkname)

			err = os.Symlink(dataDirFile, visibleFile)
			if err != nil {
				return err
			}
		}
	}
	return nil
}


// removeUserVisiblePaths removes the set of paths from the user-visible
// portion of the writer's target directory.
func (w *AtomicWriter) removeUserVisiblePaths(paths sets.String) error {
	ps := string(os.PathSeparator)
	var lasterr error
	for p := range paths {
		// only remove symlinks from the volume root directory (i.e. items that don't contain '/')
		if strings.Contains(p, ps) {
			continue
		}
		if err := os.Remove(filepath.Join(w.targetDir, p)); err != nil {
			klog.Errorf("%s: error pruning old user-visible path %s: %v", w.logContext, p, err)
			lasterr = err
		}
	}

	return lasterr
}


// GetCSIAttachLimitKey returns limit key used for CSI volumes
func GetCSIAttachLimitKey(driverName string) string {
	csiPrefixLength := len(CSIAttachLimitPrefix)
	totalkeyLength := csiPrefixLength + len(driverName)
	if totalkeyLength >= ResourceNameLengthLimit {
		charsFromDriverName := driverName[:23]
		hash := sha1.New()
		hash.Write([]byte(driverName))
		hashed := hex.EncodeToString(hash.Sum(nil))
		hashed = hashed[:16]
		return CSIAttachLimitPrefix + charsFromDriverName + hashed
	}
	return CSIAttachLimitPrefix + driverName
}


// NewDeviceHandler Create a new IoHandler implementation
func NewDeviceHandler(io IoUtil) DeviceUtil {
	return &deviceHandler{getIo: io}
}


// FindMultipathDeviceForDevice given a device name like /dev/sdx, find the devicemapper parent
func (handler *deviceHandler) FindMultipathDeviceForDevice(device string) string {
	io := handler.getIo
	disk, err := findDeviceForPath(device, io)
	if err != nil {
		return ""
	}
	sysPath := "/sys/block/"
	if dirs, err := io.ReadDir(sysPath); err == nil {
		for _, f := range dirs {
			name := f.Name()
			if strings.HasPrefix(name, "dm-") {
				if _, err1 := io.Lstat(sysPath + name + "/slaves/" + disk); err1 == nil {
					return "/dev/" + name
				}
			}
		}
	}
	return ""
}


// findDeviceForPath Find the underlying disk for a linked path such as /dev/disk/by-path/XXXX or /dev/mapper/XXXX
// will return sdX or hdX etc, if /dev/sdX is passed in then sdX will be returned
func findDeviceForPath(path string, io IoUtil) (string, error) {
	devicePath, err := io.EvalSymlinks(path)
	if err != nil {
		return "", err
	}
	// if path /dev/hdX split into "", "dev", "hdX" then we will
	// return just the last part
	parts := strings.Split(devicePath, "/")
	if len(parts) == 3 && strings.HasPrefix(parts[1], "dev") {
		return parts[2], nil
	}
	return "", errors.New("Illegal path for device " + devicePath)
}


// FindSlaveDevicesOnMultipath given a dm name like /dev/dm-1, find all devices
// which are managed by the devicemapper dm-1.
func (handler *deviceHandler) FindSlaveDevicesOnMultipath(dm string) []string {
	var devices []string
	io := handler.getIo
	// Split path /dev/dm-1 into "", "dev", "dm-1"
	parts := strings.Split(dm, "/")
	if len(parts) != 3 || !strings.HasPrefix(parts[1], "dev") {
		return devices
	}
	disk := parts[2]
	slavesPath := filepath.Join("/sys/block/", disk, "/slaves/")
	if files, err := io.ReadDir(slavesPath); err == nil {
		for _, f := range files {
			devices = append(devices, filepath.Join("/dev/", f.Name()))
		}
	}
	return devices
}


// GetISCSIPortalHostMapForTarget given a target iqn, find all the scsi hosts logged into
// that target. Returns a map of iSCSI portals (string) to SCSI host numbers (integers).
//
//	For example: {
//	   "192.168.30.7:3260": 2,
//	   "192.168.30.8:3260": 3,
//	}
func (handler *deviceHandler) GetISCSIPortalHostMapForTarget(targetIqn string) (map[string]int, error) {
	portalHostMap := make(map[string]int)
	io := handler.getIo

	// Iterate over all the iSCSI hosts in sysfs
	sysPath := "/sys/class/iscsi_host"
	hostDirs, err := io.ReadDir(sysPath)
	if err != nil {
		if os.IsNotExist(err) {
			return portalHostMap, nil
		}
		return nil, err
	}
	for _, hostDir := range hostDirs {
		// iSCSI hosts are always of the format "host%d"
		// See drivers/scsi/hosts.c in Linux
		hostName := hostDir.Name()
		if !strings.HasPrefix(hostName, "host") {
			continue
		}
		hostNumber, err := strconv.Atoi(strings.TrimPrefix(hostName, "host"))
		if err != nil {
			klog.Errorf("Could not get number from iSCSI host: %s", hostName)
			continue
		}

		// Iterate over the children of the iscsi_host device
		// We are looking for the associated session
		devicePath := sysPath + "/" + hostName + "/device"
		deviceDirs, err := io.ReadDir(devicePath)
		if err != nil {
			return nil, err
		}
		for _, deviceDir := range deviceDirs {
			// Skip over files that aren't the session
			// Sessions are of the format "session%u"
			// See drivers/scsi/scsi_transport_iscsi.c in Linux
			sessionName := deviceDir.Name()
			if !strings.HasPrefix(sessionName, "session") {
				continue
			}

			sessionPath := devicePath + "/" + sessionName

			// Read the target name for the iSCSI session
			targetNamePath := sessionPath + "/iscsi_session/" + sessionName + "/targetname"
			targetName, err := io.ReadFile(targetNamePath)
			if err != nil {
				klog.Infof("Failed to process session %s, assuming this session is unavailable: %s", sessionName, err)
				continue
			}

			// Ignore hosts that don't matchthe target we were looking for.
			if strings.TrimSpace(string(targetName)) != targetIqn {
				continue
			}

			// Iterate over the children of the iSCSI session looking
			// for the iSCSI connection.
			dirs2, err := io.ReadDir(sessionPath)
			if err != nil {
				klog.Infof("Failed to process session %s, assuming this session is unavailable: %s", sessionName, err)
				continue
			}
			for _, dir2 := range dirs2 {
				// Skip over files that aren't the connection
				// Connections are of the format "connection%d:%u"
				// See drivers/scsi/scsi_transport_iscsi.c in Linux
				dirName := dir2.Name()
				if !strings.HasPrefix(dirName, "connection") {
					continue
				}

				connectionPath := sessionPath + "/" + dirName + "/iscsi_connection/" + dirName

				// Read the current and persistent portal information for the connection.
				addrPath := connectionPath + "/address"
				addr, err := io.ReadFile(addrPath)
				if err != nil {
					klog.Infof("Failed to process connection %s, assuming this connection is unavailable: %s", dirName, err)
					continue
				}

				portPath := connectionPath + "/port"
				port, err := io.ReadFile(portPath)
				if err != nil {
					klog.Infof("Failed to process connection %s, assuming this connection is unavailable: %s", dirName, err)
					continue
				}

				persistentAddrPath := connectionPath + "/persistent_address"
				persistentAddr, err := io.ReadFile(persistentAddrPath)
				if err != nil {
					klog.Infof("Failed to process connection %s, assuming this connection is unavailable: %s", dirName, err)
					continue
				}

				persistentPortPath := connectionPath + "/persistent_port"
				persistentPort, err := io.ReadFile(persistentPortPath)
				if err != nil {
					klog.Infof("Failed to process connection %s, assuming this connection is unavailable: %s", dirName, err)
					continue
				}

				// Add entries to the map for both the current and persistent portals
				// pointing to the SCSI host for those connections
				// JoinHostPort will add `[]` around IPv6 addresses.
				portal := net.JoinHostPort(strings.TrimSpace(string(addr)), strings.TrimSpace(string(port)))
				portalHostMap[portal] = hostNumber

				persistentPortal := net.JoinHostPort(strings.TrimSpace(string(persistentAddr)), strings.TrimSpace(string(persistentPort)))
				portalHostMap[persistentPortal] = hostNumber
			}
		}
	}

	return portalHostMap, nil
}


// FindDevicesForISCSILun given an iqn, and lun number, find all the devices
// corresponding to that LUN.
func (handler *deviceHandler) FindDevicesForISCSILun(targetIqn string, lun int) ([]string, error) {
	devices := make([]string, 0)
	io := handler.getIo

	// Iterate over all the iSCSI hosts in sysfs
	sysPath := "/sys/class/iscsi_host"
	hostDirs, err := io.ReadDir(sysPath)
	if err != nil {
		return nil, err
	}
	for _, hostDir := range hostDirs {
		// iSCSI hosts are always of the format "host%d"
		// See drivers/scsi/hosts.c in Linux
		hostName := hostDir.Name()
		if !strings.HasPrefix(hostName, "host") {
			continue
		}
		hostNumber, err := strconv.Atoi(strings.TrimPrefix(hostName, "host"))
		if err != nil {
			klog.Errorf("Could not get number from iSCSI host: %s", hostName)
			continue
		}

		// Iterate over the children of the iscsi_host device
		// We are looking for the associated session
		devicePath := sysPath + "/" + hostName + "/device"
		deviceDirs, err := io.ReadDir(devicePath)
		if err != nil {
			return nil, err
		}
		for _, deviceDir := range deviceDirs {
			// Skip over files that aren't the session
			// Sessions are of the format "session%u"
			// See drivers/scsi/scsi_transport_iscsi.c in Linux
			sessionName := deviceDir.Name()
			if !strings.HasPrefix(sessionName, "session") {
				continue
			}

			// Read the target name for the iSCSI session
			targetNamePath := devicePath + "/" + sessionName + "/iscsi_session/" + sessionName + "/targetname"
			targetName, err := io.ReadFile(targetNamePath)
			if err != nil {
				return nil, err
			}

			// Only if the session matches the target we were looking for,
			// add it to the map
			if strings.TrimSpace(string(targetName)) != targetIqn {
				continue
			}

			// The list of block devices on the scsi bus will be in a
			// directory called "target%d:%d:%d".
			// See drivers/scsi/scsi_scan.c in Linux
			// We assume the channel/bus and device/controller are always zero for iSCSI
			targetPath := devicePath + "/" + sessionName + fmt.Sprintf("/target%d:0:0", hostNumber)

			// The block device for a given lun will be "%d:%d:%d:%d" --
			// host:channel:bus:LUN
			blockDevicePath := targetPath + fmt.Sprintf("/%d:0:0:%d", hostNumber, lun)

			// If the LUN doesn't exist on this bus, continue on
			_, err = io.Lstat(blockDevicePath)
			if err != nil {
				continue
			}

			// Read the block directory, there should only be one child --
			// the block device "sd*"
			path := blockDevicePath + "/block"
			dirs, err := io.ReadDir(path)
			if err != nil {
				return nil, err
			}
			if 0 < len(dirs) {
				devices = append(devices, dirs[0].Name())
			}
		}
	}

	return devices, nil
}


// FindMultipathDeviceForDevice unsupported returns ""
func (handler *deviceHandler) FindMultipathDeviceForDevice(device string) string {
	return ""
}


// FindSlaveDevicesOnMultipath unsupported returns ""
func (handler *deviceHandler) FindSlaveDevicesOnMultipath(disk string) []string {
	out := []string{}
	return out
}


// GetISCSIPortalHostMapForTarget unsupported returns nil
func (handler *deviceHandler) GetISCSIPortalHostMapForTarget(targetIqn string) (map[string]int, error) {
	portalHostMap := make(map[string]int)
	return portalHostMap, nil
}


// FindDevicesForISCSILun unsupported returns nil
func (handler *deviceHandler) FindDevicesForISCSILun(targetIqn string, lun int) ([]string, error) {
	devices := []string{}
	return devices, nil
}


// Info linux returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error)
// for the filesystem that path resides upon.
func Info(path string) (int64, int64, int64, int64, int64, int64, error) {
	statfs := &unix.Statfs_t{}
	err := unix.Statfs(path, statfs)
	if err != nil {
		return 0, 0, 0, 0, 0, 0, err
	}

	// Available is blocks available * fragment size
	available := int64(statfs.Bavail) * int64(statfs.Bsize)

	// Capacity is total block count * fragment size
	capacity := int64(statfs.Blocks) * int64(statfs.Bsize)

	// Usage is block being used * fragment size (aka block size).
	usage := (int64(statfs.Blocks) - int64(statfs.Bfree)) * int64(statfs.Bsize)

	inodes := int64(statfs.Files)
	inodesFree := int64(statfs.Ffree)
	inodesUsed := inodes - inodesFree

	return available, capacity, usage, inodes, inodesFree, inodesUsed, nil
}


// DiskUsage calculates the number of inodes and disk usage for a given directory
func DiskUsage(path string) (UsageInfo, error) {
	var usage UsageInfo

	if path == "" {
		return usage, fmt.Errorf("invalid directory")
	}

	// First check whether the quota system knows about this directory
	// A nil quantity or error means that the path does not support quotas
	// or xfs_quota tool is missing and we should use other mechanisms.
	startTime := time.Now()
	consumption, _ := fsquota.GetConsumption(path)
	if consumption != nil {
		usage.Bytes = consumption.Value()
		defer servermetrics.CollectVolumeStatCalDuration("fsquota", startTime)
	} else {
		defer servermetrics.CollectVolumeStatCalDuration("du", startTime)
	}

	inodes, _ := fsquota.GetInodes(path)
	if inodes != nil {
		usage.Inodes = inodes.Value()
	}

	if inodes != nil && consumption != nil {
		return usage, nil
	}

	topLevelStat := &unix.Stat_t{}
	err := unix.Stat(path, topLevelStat)
	if err != nil {
		return usage, err
	}

	// dedupedInode stores inodes that could be duplicates (nlink > 1)
	dedupedInodes := make(map[uint64]struct{})

	err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
		// ignore files that have been deleted after directory was read
		if os.IsNotExist(err) {
			return nil
		}
		if err != nil {
			return fmt.Errorf("unable to count inodes for %s: %s", path, err)
		}

		// according to the docs, Sys can be nil
		if info.Sys() == nil {
			return fmt.Errorf("fileinfo Sys is nil")
		}

		s, ok := info.Sys().(*syscall.Stat_t)
		if !ok {
			return fmt.Errorf("unsupported fileinfo; could not convert to stat_t")
		}

		if s.Dev != topLevelStat.Dev {
			// don't descend into directories on other devices
			return filepath.SkipDir
		}

		// Dedupe hardlinks
		if s.Nlink > 1 {
			if _, ok := dedupedInodes[s.Ino]; !ok {
				dedupedInodes[s.Ino] = struct{}{}
			} else {
				return nil
			}
		}

		if consumption == nil {
			usage.Bytes += int64(s.Blocks) * int64(512) // blocksize in bytes
		}

		if inodes == nil {
			usage.Inodes++
		}

		return nil
	})

	return usage, err
}


// Info unsupported returns 0 values for available and capacity and an error.
func Info(path string) (int64, int64, int64, int64, int64, int64, error) {
	return 0, 0, 0, 0, 0, 0, fmt.Errorf("fsinfo not supported for this build")
}


// DiskUsage gets disk usage of specified path.
func DiskUsage(path string) (UsageInfo, error) {
	var usage UsageInfo
	return usage, fmt.Errorf("directory disk usage not supported for this build.")
}


// Info returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error)
// for the filesystem that path resides upon.
func Info(path string) (int64, int64, int64, int64, int64, int64, error) {
	var freeBytesAvailable, totalNumberOfBytes, totalNumberOfFreeBytes int64
	var err error

	// The equivalent linux call supports calls against files but the syscall for windows
	// fails for files with error code: The directory name is invalid. (#99173)
	// https://docs.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499-
	// By always ensuring the directory path we meet all uses cases of this function
	path = filepath.Dir(path)
	ret, _, err := syscall.Syscall6(
		procGetDiskFreeSpaceEx.Addr(),
		4,
		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
		uintptr(unsafe.Pointer(&freeBytesAvailable)),
		uintptr(unsafe.Pointer(&totalNumberOfBytes)),
		uintptr(unsafe.Pointer(&totalNumberOfFreeBytes)),
		0,
		0,
	)
	if ret == 0 {
		return 0, 0, 0, 0, 0, 0, err
	}

	return freeBytesAvailable, totalNumberOfBytes, totalNumberOfBytes - freeBytesAvailable, 0, 0, 0, nil
}


// DiskUsage gets disk usage of specified path.
func DiskUsage(path string) (UsageInfo, error) {
	var usage UsageInfo
	info, err := os.Lstat(path)
	if err != nil {
		return usage, err
	}

	usage.Bytes, err = diskUsage(path, info)
	return usage, err
}


func diskUsage(currPath string, info os.FileInfo) (int64, error) {
	var size int64

	if info.Mode()&os.ModeSymlink != 0 {
		return size, nil
	}

	size += info.Size()

	if !info.IsDir() {
		return size, nil
	}

	dir, err := os.Open(currPath)
	if err != nil {
		return size, err
	}
	defer dir.Close()

	files, err := dir.Readdir(-1)
	if err != nil {
		return size, err
	}

	for _, file := range files {
		if file.IsDir() {
			s, err := diskUsage(filepath.Join(currPath, file.Name()), file)
			if err != nil {
				return size, err
			}
			size += s
		} else {
			size += file.Size()
		}
	}
	return size, nil
}


// GetQuotaApplier -- does this backing device support quotas that
// can be applied to directories?
func (*VolumeProvider) GetQuotaApplier(mountpoint string, backingDev string) LinuxVolumeQuotaApplier {
	for _, fsType := range linuxSupportedFilesystems {
		if isFilesystemOfType(mountpoint, backingDev, fsType.typeMagic) {
			return linuxVolumeQuotaApplier{mountpoint: mountpoint,
				maxQuota:         fsType.maxQuota,
				allowEmptyOutput: fsType.allowEmptyOutput,
			}
		}
	}
	return nil
}


func getXFSQuotaCmd() (string, error)

func getXFSQuotaCmd() (string, error) {
	quotaCmdLock.Lock()
	defer quotaCmdLock.Unlock()
	if quotaCmdInitialized {
		return quotaCmd, nil
	}
	for _, program := range quotaCmds {
		fileinfo, err := os.Stat(program)
		if err == nil && ((fileinfo.Mode().Perm() & (1 << 6)) != 0) {
			klog.V(3).Infof("Found xfs_quota program %s", program)
			quotaCmd = program
			quotaCmdInitialized = true
			return quotaCmd, nil
		}
	}
	quotaCmdInitialized = true
	return "", fmt.Errorf("no xfs_quota program found")
}


func doRunXFSQuotaCommand(mountpoint string, mountsFile, command string) (string, error) {
	quotaCmd, err := getXFSQuotaCmd()
	if err != nil {
		return "", err
	}
	// We're using numeric project IDs directly; no need to scan
	// /etc/projects or /etc/projid
	klog.V(4).Infof("runXFSQuotaCommand %s -t %s -P/dev/null -D/dev/null -x -f %s -c %s", quotaCmd, mountsFile, mountpoint, command)
	cmd := exec.Command(quotaCmd, "-t", mountsFile, "-P/dev/null", "-D/dev/null", "-x", "-f", mountpoint, "-c", command)

	data, err := cmd.Output()
	if err != nil {
		return "", err
	}
	klog.V(4).Infof("runXFSQuotaCommand output %q", string(data))
	return string(data), nil
}


// Extract the mountpoint we care about into a temporary mounts file so that xfs_quota does
// not attempt to scan every mount on the filesystem, which could hang if e. g.
// a stuck NFS mount is present.
// See https://bugzilla.redhat.com/show_bug.cgi?id=237120 for an example
// of the problem that could be caused if this were to happen.
func runXFSQuotaCommand(mountpoint string, command string) (string, error) {
	tmpMounts, err := ioutil.TempFile("", "mounts")
	if err != nil {
		return "", fmt.Errorf("cannot create temporary mount file: %v", err)
	}
	tmpMountsFileName := tmpMounts.Name()
	defer tmpMounts.Close()
	defer os.Remove(tmpMountsFileName)

	mounts, err := os.Open(MountsFile)
	if err != nil {
		return "", fmt.Errorf("cannot open mounts file %s: %v", MountsFile, err)
	}
	defer mounts.Close()

	scanner := bufio.NewScanner(mounts)
	for scanner.Scan() {
		match := MountParseRegexp.FindStringSubmatch(scanner.Text())
		if match != nil {
			mount := match[2]
			if mount == mountpoint {
				if _, err := tmpMounts.WriteString(fmt.Sprintf("%s\n", scanner.Text())); err != nil {
					return "", fmt.Errorf("cannot write temporary mounts file: %v", err)
				}
				if err := tmpMounts.Sync(); err != nil {
					return "", fmt.Errorf("cannot sync temporary mounts file: %v", err)
				}
				return doRunXFSQuotaCommand(mountpoint, tmpMountsFileName, command)
			}
		}
	}
	return "", fmt.Errorf("cannot run xfs_quota: cannot find mount point %s in %s", mountpoint, MountsFile)
}


// SupportsQuotas determines whether the filesystem supports quotas.
func SupportsQuotas(mountpoint string, qType QuotaType) (bool, error) {
	data, err := runXFSQuotaCommand(mountpoint, "state -p")
	if err != nil {
		return false, err
	}
	if qType == FSQuotaEnforcing {
		return strings.Contains(data, "Enforcement: ON"), nil
	}
	return strings.Contains(data, "Accounting: ON"), nil
}


func isFilesystemOfType(mountpoint string, backingDev string, typeMagic int64) bool {
	var buf syscall.Statfs_t
	err := syscall.Statfs(mountpoint, &buf)
	if err != nil {
		klog.Warningf("Warning: Unable to statfs %s: %v", mountpoint, err)
		return false
	}
	if int64(buf.Type) != typeMagic {
		return false
	}
	if answer, _ := SupportsQuotas(mountpoint, FSQuotaAccounting); answer {
		return true
	}
	return false
}


// GetQuotaOnDir retrieves the quota ID (if any) associated with the specified directory
// If we can't make system calls, all we can say is that we don't know whether
// it has a quota, and higher levels have to make the call.
func (v linuxVolumeQuotaApplier) GetQuotaOnDir(path string) (QuotaID, error) {
	cmd := exec.Command(lsattrCmd, "-pd", path)
	data, err := cmd.Output()
	if err != nil {
		return BadQuotaID, fmt.Errorf("cannot run lsattr: %v", err)
	}
	match := lsattrParseRegexp.FindStringSubmatch(string(data))
	if match == nil {
		return BadQuotaID, fmt.Errorf("unable to parse lsattr -pd %s output %s", path, string(data))
	}
	if match[2] != path {
		return BadQuotaID, fmt.Errorf("mismatch between supplied and returned path (%s != %s)", path, match[2])
	}
	projid, err := strconv.ParseInt(match[1], 10, 32)
	if err != nil {
		return BadQuotaID, fmt.Errorf("unable to parse project ID from %s (%v)", match[1], err)
	}
	return QuotaID(projid), nil
}


// SetQuotaOnDir applies a quota to the specified directory under the specified mountpoint.
func (v linuxVolumeQuotaApplier) SetQuotaOnDir(path string, id QuotaID, bytes int64) error {
	if bytes < 0 || bytes > v.maxQuota {
		bytes = v.maxQuota
	}
	_, err := runXFSQuotaCommand(v.mountpoint, fmt.Sprintf("limit -p bhard=%v bsoft=%v %v", bytes, bytes, id))
	if err != nil {
		return err
	}

	_, err = runXFSQuotaCommand(v.mountpoint, fmt.Sprintf("project -s -p %s %v", path, id))
	return err
}


func getQuantity(mountpoint string, id QuotaID, xfsQuotaArg string, multiplier int64, allowEmptyOutput bool) (int64, error) {
	data, err := runXFSQuotaCommand(mountpoint, fmt.Sprintf("quota -p -N -n -v %s %v", xfsQuotaArg, id))
	if err != nil {
		return 0, fmt.Errorf("unable to run xfs_quota: %v", err)
	}
	if data == "" && allowEmptyOutput {
		return 0, nil
	}
	match := quotaParseRegexp.FindStringSubmatch(data)
	if match == nil {
		return 0, fmt.Errorf("unable to parse quota output '%s'", data)
	}
	size, err := strconv.ParseInt(match[1], 10, 64)
	if err != nil {
		return 0, fmt.Errorf("unable to parse data size '%s' from '%s': %v", match[1], data, err)
	}
	klog.V(4).Infof("getQuantity %s %d %s %d => %d %v", mountpoint, id, xfsQuotaArg, multiplier, size, err)
	return size * multiplier, nil
}


// GetConsumption returns the consumption in bytes if available via quotas
func (v linuxVolumeQuotaApplier) GetConsumption(_ string, id QuotaID) (int64, error) {
	return getQuantity(v.mountpoint, id, "-b", 1024, v.allowEmptyOutput)
}


// GetInodes returns the inodes in use if available via quotas
func (v linuxVolumeQuotaApplier) GetInodes(_ string, id QuotaID) (int64, error) {
	return getQuantity(v.mountpoint, id, "-i", 1, v.allowEmptyOutput)
}


// QuotaIDIsInUse checks whether the specified quota ID is in use on the specified
// filesystem
func (v linuxVolumeQuotaApplier) QuotaIDIsInUse(id QuotaID) (bool, error) {
	bytes, err := v.GetConsumption(v.mountpoint, id)
	if err != nil {
		return false, err
	}
	if bytes > 0 {
		return true, nil
	}
	inodes, err := v.GetInodes(v.mountpoint, id)
	return inodes > 0, err
}


func projFilesAreOK() error

func projFilesAreOK() error {
	if sf, err := os.Lstat(projectsFile); err != nil || sf.Mode().IsRegular() {
		if sf, err := os.Lstat(projidFile); err != nil || sf.Mode().IsRegular() {
			return nil
		}
		return fmt.Errorf("%s exists but is not a plain file, cannot continue", projidFile)
	}
	return fmt.Errorf("%s exists but is not a plain file, cannot continue", projectsFile)
}


func lockFile(file *os.File) error

func lockFile(file *os.File) error {
	return unix.Flock(int(file.Fd()), unix.LOCK_EX)
}


func unlockFile(file *os.File) error

func unlockFile(file *os.File) error {
	return unix.Flock(int(file.Fd()), unix.LOCK_UN)
}


// openAndLockProjectFiles opens /etc/projects and /etc/projid locked.
// Creates them if they don't exist
func openAndLockProjectFiles() (*os.File, *os.File, error) {
	// Make sure neither project-related file is a symlink!
	if err := projFilesAreOK(); err != nil {
		return nil, nil, fmt.Errorf("system project files failed verification: %v", err)
	}
	// We don't actually modify the original files; we create temporaries and
	// move them over the originals
	fProjects, err := os.OpenFile(projectsFile, os.O_RDONLY|os.O_CREATE, 0644)
	if err != nil {
		err = fmt.Errorf("unable to open %s: %v", projectsFile, err)
		return nil, nil, err
	}
	fProjid, err := os.OpenFile(projidFile, os.O_RDONLY|os.O_CREATE, 0644)
	if err == nil {
		// Check once more, to ensure nothing got changed out from under us
		if err = projFilesAreOK(); err == nil {
			err = lockFile(fProjects)
			if err == nil {
				err = lockFile(fProjid)
				if err == nil {
					return fProjects, fProjid, nil
				}
				// Nothing useful we can do if we get an error here
				err = fmt.Errorf("unable to lock %s: %v", projidFile, err)
				unlockFile(fProjects)
			} else {
				err = fmt.Errorf("unable to lock %s: %v", projectsFile, err)
			}
		} else {
			err = fmt.Errorf("system project files failed re-verification: %v", err)
		}
		fProjid.Close()
	} else {
		err = fmt.Errorf("unable to open %s: %v", projidFile, err)
	}
	fProjects.Close()
	return nil, nil, err
}


func closeProjectFiles(fProjects *os.File, fProjid *os.File) error {
	// Nothing useful we can do if either of these fail,
	// but we have to close (and thereby unlock) the files anyway.
	var err error
	var err1 error
	if fProjid != nil {
		err = fProjid.Close()
	}
	if fProjects != nil {
		err1 = fProjects.Close()
	}
	if err == nil {
		return err1
	}
	return err
}


func parseProject(l string) projectType

func parseProject(l string) projectType {
	if match := projectsParseRegexp.FindStringSubmatch(l); match != nil {
		i, err := strconv.Atoi(match[1])
		if err == nil {
			return projectType{true, common.QuotaID(i), match[2], l}
		}
	}
	return projectType{true, common.BadQuotaID, "", l}
}


func parseProjid(l string) projectType

func parseProjid(l string) projectType {
	if match := projidParseRegexp.FindStringSubmatch(l); match != nil {
		i, err := strconv.Atoi(match[2])
		if err == nil {
			return projectType{true, common.QuotaID(i), match[1], l}
		}
	}
	return projectType{true, common.BadQuotaID, "", l}
}


func parseProjFile(f *os.File, parser func(l string) projectType) []projectType {
	var answer []projectType
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		answer = append(answer, parser(scanner.Text()))
	}
	return answer
}


func readProjectFiles(projects *os.File, projid *os.File) projectsList {
	return projectsList{parseProjFile(projects, parseProject), parseProjFile(projid, parseProjid)}
}


// findAvailableQuota finds the next available quota from the FirstQuota
// it returns error if QuotaIDIsInUse returns error when getting quota id in use;
// it searches at most maxUnusedQuotasToSearch(128) time
func findAvailableQuota(path string, idMap map[common.QuotaID]bool) (common.QuotaID, error) {
	unusedQuotasSearched := 0
	for id := common.FirstQuota; true; id++ {
		if _, ok := idMap[id]; !ok {
			isInUse, err := getApplier(path).QuotaIDIsInUse(id)
			if err != nil {
				return common.BadQuotaID, err
			} else if !isInUse {
				return id, nil
			}
			unusedQuotasSearched++
			if unusedQuotasSearched > maxUnusedQuotasToSearch {
				break
			}
		}
	}
	return common.BadQuotaID, fmt.Errorf("cannot find available quota ID")
}


func addDirToProject(path string, id common.QuotaID, list *projectsList) (common.QuotaID, bool, error) {
	idMap := make(map[common.QuotaID]bool)
	for _, project := range list.projects {
		if project.data == path {
			if id != common.BadQuotaID && id != project.id {
				return common.BadQuotaID, false, fmt.Errorf("attempt to reassign project ID for %s", path)
			}
			// Trying to reassign a directory to the project it's
			// already in.  Maybe this should be an error, but for
			// now treat it as an idempotent operation
			return project.id, false, nil
		}
		idMap[project.id] = true
	}
	var needToAddProjid = true
	for _, projid := range list.projid {
		idMap[projid.id] = true
		if projid.id == id && id != common.BadQuotaID {
			needToAddProjid = false
		}
	}
	var err error
	if id == common.BadQuotaID {
		id, err = findAvailableQuota(path, idMap)
		if err != nil {
			return common.BadQuotaID, false, err
		}
		needToAddProjid = true
	}
	if needToAddProjid {
		name := fmt.Sprintf("volume%v", id)
		line := fmt.Sprintf("%s:%v", name, id)
		list.projid = append(list.projid, projectType{true, id, name, line})
	}
	line := fmt.Sprintf("%v:%s", id, path)
	list.projects = append(list.projects, projectType{true, id, path, line})
	return id, needToAddProjid, nil
}


func removeDirFromProject(path string, id common.QuotaID, list *projectsList) (bool, error) {
	if id == common.BadQuotaID {
		return false, fmt.Errorf("attempt to remove invalid quota ID from %s", path)
	}
	foundAt := -1
	countByID := make(map[common.QuotaID]int)
	for i, project := range list.projects {
		if project.data == path {
			if id != project.id {
				return false, fmt.Errorf("attempting to remove quota ID %v from path %s, but expecting ID %v", id, path, project.id)
			} else if foundAt != -1 {
				return false, fmt.Errorf("found multiple quota IDs for path %s", path)
			}
			// Faster and easier than deleting an element
			list.projects[i].isValid = false
			foundAt = i
		}
		countByID[project.id]++
	}
	if foundAt == -1 {
		return false, fmt.Errorf("cannot find quota associated with path %s", path)
	}
	if countByID[id] <= 1 {
		// Removing the last entry means that we're no longer using
		// the quota ID, so remove that as well
		for i, projid := range list.projid {
			if projid.id == id {
				list.projid[i].isValid = false
			}
		}
		return true, nil
	}
	return false, nil
}


func writeProjectFile(base *os.File, projects []projectType) (string, error) {
	oname := base.Name()
	stat, err := base.Stat()
	if err != nil {
		return "", err
	}
	mode := stat.Mode() & os.ModePerm
	f, err := ioutil.TempFile(filepath.Dir(oname), filepath.Base(oname))
	if err != nil {
		return "", err
	}
	filename := f.Name()
	if err := os.Chmod(filename, mode); err != nil {
		return "", err
	}
	for _, proj := range projects {
		if proj.isValid {
			if _, err := f.WriteString(fmt.Sprintf("%s\n", proj.line)); err != nil {
				f.Close()
				os.Remove(filename)
				return "", err
			}
		}
	}
	if err := f.Close(); err != nil {
		os.Remove(filename)
		return "", err
	}
	return filename, nil
}


func writeProjectFiles(fProjects *os.File, fProjid *os.File, writeProjid bool, list projectsList) error {
	tmpProjects, err := writeProjectFile(fProjects, list.projects)
	if err == nil {
		// Ensure that both files are written before we try to rename either.
		if writeProjid {
			tmpProjid, err := writeProjectFile(fProjid, list.projid)
			if err == nil {
				err = os.Rename(tmpProjid, fProjid.Name())
				if err != nil {
					os.Remove(tmpProjid)
				}
			}
		}
		if err == nil {
			err = os.Rename(tmpProjects, fProjects.Name())
			if err == nil {
				return nil
			}
			// We're in a bit of trouble here; at this
			// point we've successfully renamed tmpProjid
			// to the real thing, but renaming tmpProject
			// to the real file failed.  There's not much we
			// can do in this position.  Anything we could do
			// to try to undo it would itself be likely to fail.
		}
		os.Remove(tmpProjects)
	}
	return fmt.Errorf("unable to write project files: %v", err)
}


// if ID is common.BadQuotaID, generate new project id if the dir is not in a project
func createProjectID(path string, ID common.QuotaID) (common.QuotaID, error) {
	quotaIDLock.Lock()
	defer quotaIDLock.Unlock()
	fProjects, fProjid, err := openAndLockProjectFiles()
	if err == nil {
		defer closeProjectFiles(fProjects, fProjid)
		list := readProjectFiles(fProjects, fProjid)
		var writeProjid bool
		ID, writeProjid, err = addDirToProject(path, ID, &list)
		if err == nil && ID != common.BadQuotaID {
			if err = writeProjectFiles(fProjects, fProjid, writeProjid, list); err == nil {
				return ID, nil
			}
		}
	}
	return common.BadQuotaID, fmt.Errorf("createProjectID %s %v failed %v", path, ID, err)
}


func removeProjectID(path string, ID common.QuotaID) error {
	if ID == common.BadQuotaID {
		return fmt.Errorf("attempting to remove invalid quota ID %v", ID)
	}
	quotaIDLock.Lock()
	defer quotaIDLock.Unlock()
	fProjects, fProjid, err := openAndLockProjectFiles()
	if err == nil {
		defer closeProjectFiles(fProjects, fProjid)
		list := readProjectFiles(fProjects, fProjid)
		var writeProjid bool
		writeProjid, err = removeDirFromProject(path, ID, &list)
		if err == nil {
			if err = writeProjectFiles(fProjects, fProjid, writeProjid, list); err == nil {
				return nil
			}
		}
	}
	return fmt.Errorf("removeProjectID %s %v failed %v", path, ID, err)
}


func enabledQuotasForMonitoring() bool

func enabledQuotasForMonitoring() bool {
	return utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolationFSQuotaMonitoring)
}


// Separate the innards for ease of testing
func detectBackingDevInternal(mountpoint string, mounts string) (string, error) {
	file, err := os.Open(mounts)
	if err != nil {
		return "", err
	}
	defer file.Close()
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		match := common.MountParseRegexp.FindStringSubmatch(scanner.Text())
		if match != nil {
			device := match[1]
			mount := match[2]
			if mount == mountpoint {
				return device, nil
			}
		}
	}
	return "", fmt.Errorf("couldn't find backing device for %s", mountpoint)
}


// detectBackingDev assumes that the mount point provided is valid
func detectBackingDev(_ mount.Interface, mountpoint string) (string, error) {
	return detectBackingDevInternal(mountpoint, common.MountsFile)
}


func clearBackingDev(path string)

func clearBackingDev(path string) {
	backingDevLock.Lock()
	defer backingDevLock.Unlock()
	delete(backingDevMap, path)
}


// Assumes that the path has been fully canonicalized
// Breaking this up helps with testing
func detectMountpointInternal(m mount.Interface, path string) (string, error) {
	for path != "" && path != "/" {
		// per k8s.io/mount-utils/mount_linux this detects all but
		// a bind mount from one part of a mount to another.
		// For our purposes that's fine; we simply want the "true"
		// mount point
		//
		// IsNotMountPoint proved much more troublesome; it actually
		// scans the mounts, and when a lot of mount/unmount
		// activity takes place, it is not able to get a consistent
		// view of /proc/self/mounts, causing it to time out and
		// report incorrectly.
		isNotMount, err := m.IsLikelyNotMountPoint(path)
		if err != nil {
			return "/", err
		}
		if !isNotMount {
			return path, nil
		}
		path = filepath.Dir(path)
	}
	return "/", nil
}


func detectMountpoint(m mount.Interface, path string) (string, error) {
	xpath, err := filepath.Abs(path)
	if err != nil {
		return "/", err
	}
	xpath, err = filepath.EvalSymlinks(xpath)
	if err != nil {
		return "/", err
	}
	if xpath, err = detectMountpointInternal(m, xpath); err == nil {
		return xpath, nil
	}
	return "/", err
}


func clearMountpoint(path string)

func clearMountpoint(path string) {
	mountpointLock.Lock()
	defer mountpointLock.Unlock()
	delete(mountpointMap, path)
}


// getFSInfo Returns mountpoint and backing device
// getFSInfo should cache the mountpoint and backing device for the
// path.
func getFSInfo(m mount.Interface, path string) (string, string, error) {
	mountpointLock.Lock()
	defer mountpointLock.Unlock()

	backingDevLock.Lock()
	defer backingDevLock.Unlock()

	var err error

	mountpoint, okMountpoint := mountpointMap[path]
	if !okMountpoint {
		mountpoint, err = detectMountpoint(m, path)
		if err != nil {
			return "", "", fmt.Errorf("cannot determine mountpoint for %s: %v", path, err)
		}
	}

	backingDev, okBackingDev := backingDevMap[path]
	if !okBackingDev {
		backingDev, err = detectBackingDev(m, mountpoint)
		if err != nil {
			return "", "", fmt.Errorf("cannot determine backing device for %s: %v", path, err)
		}
	}
	mountpointMap[path] = mountpoint
	backingDevMap[path] = backingDev
	return mountpoint, backingDev, nil
}


func clearFSInfo(path string)

func clearFSInfo(path string) {
	clearMountpoint(path)
	clearBackingDev(path)
}


func getApplier(path string) common.LinuxVolumeQuotaApplier {
	dirApplierLock.Lock()
	defer dirApplierLock.Unlock()
	return dirApplierMap[path]
}


func setApplier(path string, applier common.LinuxVolumeQuotaApplier) {
	dirApplierLock.Lock()
	defer dirApplierLock.Unlock()
	dirApplierMap[path] = applier
}


func clearApplier(path string)

func clearApplier(path string) {
	dirApplierLock.Lock()
	defer dirApplierLock.Unlock()
	delete(dirApplierMap, path)
}


func setQuotaOnDir(path string, id common.QuotaID, bytes int64) error {
	return getApplier(path).SetQuotaOnDir(path, id, bytes)
}


func GetQuotaOnDir(m mount.Interface, path string) (common.QuotaID, error) {
	_, _, err := getFSInfo(m, path)
	if err != nil {
		return common.BadQuotaID, err
	}
	return getApplier(path).GetQuotaOnDir(path)
}


func clearQuotaOnDir(m mount.Interface, path string) error {
	// Since we may be called without path being in the map,
	// we explicitly have to check in this case.
	klog.V(4).Infof("clearQuotaOnDir %s", path)
	supportsQuotas, err := SupportsQuotas(m, path)
	if err != nil {
		// Log-and-continue instead of returning an error for now
		// due to unspecified backwards compatibility concerns (a subject to revise)
		klog.V(3).Infof("Attempt to check for quota support failed: %v", err)
	}
	if !supportsQuotas {
		return nil
	}
	projid, err := GetQuotaOnDir(m, path)
	if err == nil && projid != common.BadQuotaID {
		// This means that we have a quota on the directory but
		// we can't clear it.  That's not good.
		err = setQuotaOnDir(path, projid, 0)
		if err != nil {
			klog.V(3).Infof("Attempt to clear quota failed: %v", err)
		}
		// Even if clearing the quota failed, we still need to
		// try to remove the project ID, or that may be left dangling.
		err1 := removeProjectID(path, projid)
		if err1 != nil {
			klog.V(3).Infof("Attempt to remove quota ID from system files failed: %v", err1)
		}
		clearFSInfo(path)
		if err != nil {
			return err
		}
		return err1
	}
	// If we couldn't get a quota, that's fine -- there may
	// never have been one, and we have no way to know otherwise
	klog.V(3).Infof("clearQuotaOnDir fails %v", err)
	return nil
}


// SupportsQuotas -- Does the path support quotas
// Cache the applier for paths that support quotas.  For paths that don't,
// don't cache the result because nothing will clean it up.
// However, do cache the device->applier map; the number of devices
// is bounded.
func SupportsQuotas(m mount.Interface, path string) (bool, error) {
	if !enabledQuotasForMonitoring() {
		klog.V(3).Info("SupportsQuotas called, but quotas disabled")
		return false, nil
	}
	supportsQuotasLock.Lock()
	defer supportsQuotasLock.Unlock()
	if supportsQuotas, ok := supportsQuotasMap[path]; ok {
		return supportsQuotas, nil
	}
	mount, dev, err := getFSInfo(m, path)
	if err != nil {
		return false, err
	}
	// Do we know about this device?
	applier, ok := devApplierMap[mount]
	if !ok {
		for _, provider := range providers {
			if applier = provider.GetQuotaApplier(mount, dev); applier != nil {
				devApplierMap[mount] = applier
				break
			}
		}
	}
	if applier != nil {
		supportsQuotasMap[path] = true
		setApplier(path, applier)
		return true, nil
	}
	delete(backingDevMap, path)
	delete(mountpointMap, path)
	return false, nil
}


// AssignQuota -- assign a quota to the specified directory.
// AssignQuota chooses the quota ID based on the pod UID and path.
// If the pod UID is identical to another one known, it may (but presently
// doesn't) choose the same quota ID as other volumes in the pod.
func AssignQuota(m mount.Interface, path string, poduid types.UID, bytes *resource.Quantity) error { //nolint:staticcheck
	if bytes == nil {
		return fmt.Errorf("attempting to assign null quota to %s", path)
	}
	ibytes := bytes.Value()
	if ok, err := SupportsQuotas(m, path); !ok {
		return fmt.Errorf("quotas not supported on %s: %v", path, err)
	}
	quotaLock.Lock()
	defer quotaLock.Unlock()
	// Current policy is to set individual quotas on each volume,
	// for each new volume we generate a random UUID and we use that as
	// the internal pod uid.
	// From fsquota point of view each volume is attached to a
	// single unique pod.
	// If we decide later that we want to assign one quota for all
	// volumes in a pod, we can simply use poduid parameter directly
	// If and when we decide permanently that we're going to adopt
	// one quota per volume, we can rip all of the pod code out.
	externalPodUid := poduid
	internalPodUid, ok := dirPodMap[path]
	if ok {
		if podUidMap[internalPodUid] != externalPodUid {
			return fmt.Errorf("requesting quota on existing directory %s but different pod %s %s", path, podUidMap[internalPodUid], externalPodUid)
		}
	} else {
		internalPodUid = types.UID(uuid.NewUUID())
	}
	oid, ok := podQuotaMap[internalPodUid]
	if ok {
		if quotaSizeMap[oid] != ibytes {
			return fmt.Errorf("requesting quota of different size: old %v new %v", quotaSizeMap[oid], bytes)
		}
		if _, ok := dirPodMap[path]; ok {
			return nil
		}
	} else {
		oid = common.BadQuotaID
	}
	id, err := createProjectID(path, oid)
	if err == nil {
		if oid != common.BadQuotaID && oid != id {
			return fmt.Errorf("attempt to reassign quota %v to %v", oid, id)
		}
		// When enforcing quotas are enabled, we'll condition this
		// on their being disabled also.
		if ibytes > 0 {
			ibytes = -1
		}
		if err = setQuotaOnDir(path, id, ibytes); err == nil {
			quotaPodMap[id] = internalPodUid
			quotaSizeMap[id] = ibytes
			podQuotaMap[internalPodUid] = id
			dirQuotaMap[path] = id
			dirPodMap[path] = internalPodUid
			podUidMap[internalPodUid] = externalPodUid
			podDirCountMap[internalPodUid]++
			klog.V(4).Infof("Assigning quota ID %d (%d) to %s", id, ibytes, path)
			return nil
		}
		removeProjectID(path, id)
	}
	return fmt.Errorf("assign quota FAILED %v", err)
}


// GetConsumption -- retrieve the consumption (in bytes) of the directory
func GetConsumption(path string) (*resource.Quantity, error) {
	// Note that we actually need to hold the lock at least through
	// running the quota command, so it can't get recycled behind our back
	quotaLock.Lock()
	defer quotaLock.Unlock()
	applier := getApplier(path)
	// No applier means directory is not under quota management
	if applier == nil {
		return nil, nil
	}
	ibytes, err := applier.GetConsumption(path, dirQuotaMap[path])
	if err != nil {
		return nil, err
	}
	return resource.NewQuantity(ibytes, resource.DecimalSI), nil
}


// GetInodes -- retrieve the number of inodes in use under the directory
func GetInodes(path string) (*resource.Quantity, error) {
	// Note that we actually need to hold the lock at least through
	// running the quota command, so it can't get recycled behind our back
	quotaLock.Lock()
	defer quotaLock.Unlock()
	applier := getApplier(path)
	// No applier means directory is not under quota management
	if applier == nil {
		return nil, nil
	}
	inodes, err := applier.GetInodes(path, dirQuotaMap[path])
	if err != nil {
		return nil, err
	}
	return resource.NewQuantity(inodes, resource.DecimalSI), nil
}


// ClearQuota -- remove the quota assigned to a directory
func ClearQuota(m mount.Interface, path string) error {
	klog.V(3).Infof("ClearQuota %s", path)
	if !enabledQuotasForMonitoring() {
		return fmt.Errorf("clearQuota called, but quotas disabled")
	}
	quotaLock.Lock()
	defer quotaLock.Unlock()
	poduid, ok := dirPodMap[path]
	if !ok {
		// Nothing in the map either means that there was no
		// quota to begin with or that we're clearing a
		// stale directory, so if we find a quota, just remove it.
		// The process of clearing the quota requires that an applier
		// be found, which needs to be cleaned up.
		defer delete(supportsQuotasMap, path)
		defer clearApplier(path)
		return clearQuotaOnDir(m, path)
	}
	_, ok = podQuotaMap[poduid]
	if !ok {
		return fmt.Errorf("clearQuota: No quota available for %s", path)
	}
	projid, err := GetQuotaOnDir(m, path)
	if err != nil {
		// Log-and-continue instead of returning an error for now
		// due to unspecified backwards compatibility concerns (a subject to revise)
		klog.V(3).Infof("Attempt to check quota ID %v on dir %s failed: %v", dirQuotaMap[path], path, err)
	}
	if projid != dirQuotaMap[path] {
		return fmt.Errorf("expected quota ID %v on dir %s does not match actual %v", dirQuotaMap[path], path, projid)
	}
	count, ok := podDirCountMap[poduid]
	if count <= 1 || !ok {
		err = clearQuotaOnDir(m, path)
		// This error should be noted; we still need to clean up
		// and otherwise handle in the same way.
		if err != nil {
			klog.V(3).Infof("Unable to clear quota %v %s: %v", dirQuotaMap[path], path, err)
		}
		delete(quotaSizeMap, podQuotaMap[poduid])
		delete(quotaPodMap, podQuotaMap[poduid])
		delete(podDirCountMap, poduid)
		delete(podQuotaMap, poduid)
		delete(podUidMap, poduid)
	} else {
		err = removeProjectID(path, projid)
		podDirCountMap[poduid]--
		klog.V(4).Infof("Not clearing quota for pod %s; still %v dirs outstanding", poduid, podDirCountMap[poduid])
	}
	delete(dirPodMap, path)
	delete(dirQuotaMap, path)
	delete(supportsQuotasMap, path)
	clearApplier(path)
	if err != nil {
		return fmt.Errorf("unable to clear quota for %s: %v", path, err)
	}
	return nil
}


func GetQuotaOnDir(_ mount.Interface, _ string) (common.QuotaID, error) {
	return common.BadQuotaID, errNotImplemented
}


// SupportsQuotas -- dummy implementation
func SupportsQuotas(_ mount.Interface, _ string) (bool, error) {
	return false, errNotImplemented
}


// AssignQuota -- dummy implementation
func AssignQuota(_ mount.Interface, _ string, _ types.UID, _ *resource.Quantity) error {
	return errNotImplemented
}


// GetConsumption -- dummy implementation
func GetConsumption(_ string) (*resource.Quantity, error) {
	return nil, errNotImplemented
}


// GetInodes -- dummy implementation
func GetInodes(_ string) (*resource.Quantity, error) {
	return nil, errNotImplemented
}


// ClearQuota -- dummy implementation
func ClearQuota(_ mount.Interface, _ string) error {
	return errNotImplemented
}


// getFileType checks for file/directory/socket and block/character devices.
func getFileType(pathname string) (FileType, error) {
	var pathType FileType
	info, err := os.Stat(pathname)
	if os.IsNotExist(err) {
		return pathType, fmt.Errorf("path %q does not exist", pathname)
	}
	// err in call to os.Stat
	if err != nil {
		return pathType, err
	}

	// checks whether the mode is the target mode.
	isSpecificMode := func(mode, targetMode os.FileMode) bool {
		return mode&targetMode == targetMode
	}

	mode := info.Mode()
	if mode.IsDir() {
		return FileTypeDirectory, nil
	} else if mode.IsRegular() {
		return FileTypeFile, nil
	} else if isSpecificMode(mode, os.ModeSocket) {
		return FileTypeSocket, nil
	} else if isSpecificMode(mode, os.ModeDevice) {
		if isSpecificMode(mode, os.ModeCharDevice) {
			return FileTypeCharDev, nil
		}
		return FileTypeBlockDev, nil
	}

	return pathType, fmt.Errorf("only recognise file, directory, socket, block device and character device")
}


func NewHostUtil() *HostUtil

// NewHostUtil returns a struct that implements the HostUtils interface on
// linux platforms
func NewHostUtil() *HostUtil {
	return &HostUtil{}
}


// DeviceOpened checks if block device in use by calling Open with O_EXCL flag.
// If pathname is not a device, log and return false with nil error.
// If open returns errno EBUSY, return true with nil error.
// If open returns nil, return false with nil error.
// Otherwise, return false with error
func (hu *HostUtil) DeviceOpened(pathname string) (bool, error) {
	return ExclusiveOpenFailsOnDevice(pathname)
}


// PathIsDevice uses FileInfo returned from os.Stat to check if path refers
// to a device.
func (hu *HostUtil) PathIsDevice(pathname string) (bool, error) {
	pathType, err := hu.GetFileType(pathname)
	isDevice := pathType == FileTypeCharDev || pathType == FileTypeBlockDev
	return isDevice, err
}


// ExclusiveOpenFailsOnDevice is shared with NsEnterMounter
func ExclusiveOpenFailsOnDevice(pathname string) (bool, error) {
	var isDevice bool
	finfo, err := os.Stat(pathname)
	if os.IsNotExist(err) {
		isDevice = false
	}
	// err in call to os.Stat
	if err != nil {
		return false, fmt.Errorf(
			"PathIsDevice failed for path %q: %v",
			pathname,
			err)
	}
	// path refers to a device
	if finfo.Mode()&os.ModeDevice != 0 {
		isDevice = true
	}

	if !isDevice {
		klog.Errorf("Path %q is not referring to a device.", pathname)
		return false, nil
	}
	fd, errno := unix.Open(pathname, unix.O_RDONLY|unix.O_EXCL|unix.O_CLOEXEC, 0)
	// If the device is in use, open will return an invalid fd.
	// When this happens, it is expected that Close will fail and throw an error.
	defer unix.Close(fd)
	if errno == nil {
		// device not in use
		return false, nil
	} else if errno == unix.EBUSY {
		// device is in use
		return true, nil
	}
	// error during call to Open
	return false, errno
}


// GetDeviceNameFromMount given a mount point, find the device name from its global mount point
func (hu *HostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
	return getDeviceNameFromMount(mounter, mountPath, pluginMountDir)
}


// getDeviceNameFromMountLinux find the device name from /proc/mounts in which
// the mount path reference should match the given plugin mount directory. In case no mount path reference
// matches, returns the volume name taken from its given mountPath
func getDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
	refs, err := mounter.GetMountRefs(mountPath)
	if err != nil {
		klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err)
		return "", err
	}
	if len(refs) == 0 {
		klog.V(4).Infof("Directory %s is not mounted", mountPath)
		return "", fmt.Errorf("directory %s is not mounted", mountPath)
	}
	for _, ref := range refs {
		if strings.HasPrefix(ref, pluginMountDir) {
			volumeID, err := filepath.Rel(pluginMountDir, ref)
			if err != nil {
				klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err)
				return "", err
			}
			return volumeID, nil
		}
	}

	return path.Base(mountPath), nil
}


// MakeRShared checks that given path is on a mount with 'rshared' mount
// propagation. If not, it bind-mounts the path as rshared.
func (hu *HostUtil) MakeRShared(path string) error {
	return DoMakeRShared(path, procMountInfoPath)
}


// GetFileType checks for file/directory/socket/block/character devices.
func (hu *HostUtil) GetFileType(pathname string) (FileType, error) {
	return getFileType(pathname)
}


// PathExists tests if the given path already exists
// Error is returned on any other error than "file not found".
func (hu *HostUtil) PathExists(pathname string) (bool, error) {
	return utilpath.Exists(utilpath.CheckFollowSymlink, pathname)
}


// EvalHostSymlinks returns the path name after evaluating symlinks.
// TODO once the nsenter implementation is removed, this method can be removed
// from the interface and filepath.EvalSymlinks used directly
func (hu *HostUtil) EvalHostSymlinks(pathname string) (string, error) {
	return filepath.EvalSymlinks(pathname)
}


// FindMountInfo returns the mount info on the given path.
func (hu *HostUtil) FindMountInfo(path string) (mount.MountInfo, error) {
	return findMountInfo(path, procMountInfoPath)
}


// isShared returns true, if given path is on a mount point that has shared
// mount propagation.
func isShared(mount string, mountInfoPath string) (bool, error) {
	info, err := findMountInfo(mount, mountInfoPath)
	if err != nil {
		return false, err
	}

	// parse optional parameters
	for _, opt := range info.OptionalFields {
		if strings.HasPrefix(opt, "shared:") {
			return true, nil
		}
	}
	return false, nil
}


func findMountInfo(path, mountInfoPath string) (mount.MountInfo, error) {
	infos, err := mount.ParseMountInfo(mountInfoPath)
	if err != nil {
		return mount.MountInfo{}, err
	}

	// process /proc/xxx/mountinfo in backward order and find the first mount
	// point that is prefix of 'path' - that's the mount where path resides
	var info *mount.MountInfo
	for i := len(infos) - 1; i >= 0; i-- {
		if mount.PathWithinBase(path, infos[i].MountPoint) {
			info = &infos[i]
			break
		}
	}
	if info == nil {
		return mount.MountInfo{}, fmt.Errorf("cannot find mount point for %q", path)
	}
	return *info, nil
}


// DoMakeRShared is common implementation of MakeRShared on Linux. It checks if
// path is shared and bind-mounts it as rshared if needed. mountCmd and
// mountArgs are expected to contain mount-like command, DoMakeRShared will add
// '--bind <path> <path>' and '--make-rshared <path>' to mountArgs.
func DoMakeRShared(path string, mountInfoFilename string) error {
	shared, err := isShared(path, mountInfoFilename)
	if err != nil {
		return err
	}
	if shared {
		klog.V(4).Infof("Directory %s is already on a shared mount", path)
		return nil
	}

	klog.V(2).Infof("Bind-mounting %q with shared mount propagation", path)
	// mount --bind /var/lib/kubelet /var/lib/kubelet
	if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_BIND, "" /*data*/); err != nil {
		return fmt.Errorf("failed to bind-mount %s: %v", path, err)
	}

	// mount --make-rshared /var/lib/kubelet
	if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_SHARED|syscall.MS_REC, "" /*data*/); err != nil {
		return fmt.Errorf("failed to make %s rshared: %v", path, err)
	}

	return nil
}


// GetSELinux is common implementation of GetSELinuxSupport on Linux.
func GetSELinux(path string, mountInfoFilename string, selinuxEnabled seLinuxEnabledFunc) (bool, error) {
	// Skip /proc/mounts parsing if SELinux is disabled.
	if !selinuxEnabled() {
		return false, nil
	}

	info, err := findMountInfo(path, mountInfoFilename)
	if err != nil {
		return false, err
	}

	// "seclabel" can be both in mount options and super options.
	for _, opt := range info.SuperOptions {
		if opt == "seclabel" {
			return true, nil
		}
	}
	for _, opt := range info.MountOptions {
		if opt == "seclabel" {
			return true, nil
		}
	}
	return false, nil
}


// GetSELinuxSupport returns true if given path is on a mount that supports
// SELinux.
func (hu *HostUtil) GetSELinuxSupport(pathname string) (bool, error) {
	return GetSELinux(pathname, procMountInfoPath, selinux.GetEnabled)
}


// GetOwner returns the integer ID for the user and group of the given path
func (hu *HostUtil) GetOwner(pathname string) (int64, int64, error) {
	realpath, err := filepath.EvalSymlinks(pathname)
	if err != nil {
		return -1, -1, err
	}
	return GetOwnerLinux(realpath)
}


// GetMode returns permissions of the path.
func (hu *HostUtil) GetMode(pathname string) (os.FileMode, error) {
	return GetModeLinux(pathname)
}


// GetOwnerLinux is shared between Linux and NsEnterMounter
// pathname must already be evaluated for symlinks
func GetOwnerLinux(pathname string) (int64, int64, error) {
	info, err := os.Stat(pathname)
	if err != nil {
		return -1, -1, err
	}
	stat := info.Sys().(*syscall.Stat_t)
	return int64(stat.Uid), int64(stat.Gid), nil
}


// GetModeLinux is shared between Linux and NsEnterMounter
func GetModeLinux(pathname string) (os.FileMode, error) {
	info, err := os.Stat(pathname)
	if err != nil {
		return 0, err
	}
	return info.Mode(), nil
}


// GetSELinuxMountContext returns value of -o context=XYZ mount option on
// given mount point.
func (hu *HostUtil) GetSELinuxMountContext(pathname string) (string, error) {
	return getSELinuxMountContext(pathname, procMountInfoPath, selinux.GetEnabled)
}


// getSELinux is common implementation of GetSELinuxSupport on Linux.
// Using an extra function for unit tests.
func getSELinuxMountContext(path string, mountInfoFilename string, selinuxEnabled seLinuxEnabledFunc) (string, error) {
	// Skip /proc/mounts parsing if SELinux is disabled.
	if !selinuxEnabled() {
		return "", nil
	}

	info, err := findMountInfo(path, mountInfoFilename)
	if err != nil {
		return "", err
	}

	for _, opt := range info.SuperOptions {
		if !strings.HasPrefix(opt, "context=") {
			continue
		}
		// Remove context=
		context := strings.TrimPrefix(opt, "context=")
		// Remove double quotes
		context = strings.Trim(context, "\"")
		return context, nil
	}
	return "", nil
}


// NewHostUtil returns a struct that implements the HostUtils interface on
// unsupported platforms
func NewHostUtil() *HostUtil {
	return &HostUtil{}
}


// DeviceOpened always returns an error on unsupported platforms
func (hu *HostUtil) DeviceOpened(pathname string) (bool, error) {
	return false, errUnsupported
}


// PathIsDevice always returns an error on unsupported platforms
func (hu *HostUtil) PathIsDevice(pathname string) (bool, error) {
	return true, errUnsupported
}


// GetDeviceNameFromMount always returns an error on unsupported platforms
func (hu *HostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
	return getDeviceNameFromMount(mounter, mountPath, pluginMountDir)
}


// MakeRShared always returns an error on unsupported platforms
func (hu *HostUtil) MakeRShared(path string) error {
	return errUnsupported
}


// GetFileType always returns an error on unsupported platforms
func (hu *HostUtil) GetFileType(pathname string) (FileType, error) {
	return FileType("fake"), errUnsupported
}


// MakeFile always returns an error on unsupported platforms
func (hu *HostUtil) MakeFile(pathname string) error {
	return errUnsupported
}


// MakeDir always returns an error on unsupported platforms
func (hu *HostUtil) MakeDir(pathname string) error {
	return errUnsupported
}


// PathExists always returns an error on unsupported platforms
func (hu *HostUtil) PathExists(pathname string) (bool, error) {
	return true, errUnsupported
}


// EvalHostSymlinks always returns an error on unsupported platforms
func (hu *HostUtil) EvalHostSymlinks(pathname string) (string, error) {
	return "", errUnsupported
}


// GetOwner always returns an error on unsupported platforms
func (hu *HostUtil) GetOwner(pathname string) (int64, int64, error) {
	return -1, -1, errUnsupported
}


// GetSELinuxSupport always returns an error on unsupported platforms
func (hu *HostUtil) GetSELinuxSupport(pathname string) (bool, error) {
	return false, errUnsupported
}


// GetMode always returns an error on unsupported platforms
func (hu *HostUtil) GetMode(pathname string) (os.FileMode, error) {
	return 0, errUnsupported
}


func getDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
	return "", errUnsupported
}


// GetSELinuxMountContext returns value of -o context=XYZ mount option on
// given mount point.
func (hu *HostUtil) GetSELinuxMountContext(pathname string) (string, error) {
	return "", errUnsupported
}


// NewHostUtil returns a struct that implements HostUtils on Windows platforms
func NewHostUtil() *HostUtil {
	return &HostUtil{}
}


// GetDeviceNameFromMount given a mnt point, find the device
func (hu *HostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
	return getDeviceNameFromMount(mounter, mountPath, pluginMountDir)
}


// getDeviceNameFromMount find the device(drive) name in which
// the mount path reference should match the given plugin mount directory. In case no mount path reference
// matches, returns the volume name taken from its given mountPath
func getDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
	refs, err := mounter.GetMountRefs(mountPath)
	if err != nil {
		klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err)
		return "", err
	}
	if len(refs) == 0 {
		return "", fmt.Errorf("directory %s is not mounted", mountPath)
	}
	basemountPath := mount.NormalizeWindowsPath(pluginMountDir)
	for _, ref := range refs {
		if strings.Contains(ref, basemountPath) {
			volumeID, err := filepath.Rel(mount.NormalizeWindowsPath(basemountPath), ref)
			if err != nil {
				klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err)
				return "", err
			}
			return volumeID, nil
		}
	}

	return path.Base(mountPath), nil
}


// DeviceOpened determines if the device is in use elsewhere
func (hu *HostUtil) DeviceOpened(pathname string) (bool, error) {
	return false, nil
}


// PathIsDevice determines if a path is a device.
func (hu *HostUtil) PathIsDevice(pathname string) (bool, error) {
	return false, nil
}


// MakeRShared checks that given path is on a mount with 'rshared' mount
// propagation. Empty implementation here.
func (hu *HostUtil) MakeRShared(path string) error {
	return nil
}


// GetFileType checks for sockets/block/character devices
func (hu *(HostUtil)) GetFileType(pathname string) (FileType, error) {
	return getFileType(pathname)
}


// PathExists checks whether the path exists
func (hu *HostUtil) PathExists(pathname string) (bool, error) {
	return utilpath.Exists(utilpath.CheckFollowSymlink, pathname)
}


// EvalHostSymlinks returns the path name after evaluating symlinks
func (hu *HostUtil) EvalHostSymlinks(pathname string) (string, error) {
	return filepath.EvalSymlinks(pathname)
}


// GetOwner returns the integer ID for the user and group of the given path
// Note that on windows, it always returns 0. We actually don't set Group on
// windows platform, see SetVolumeOwnership implementation.
func (hu *HostUtil) GetOwner(pathname string) (int64, int64, error) {
	return -1, -1, nil
}


// GetSELinuxSupport returns a boolean indicating support for SELinux.
// Windows does not support SELinux.
func (hu *HostUtil) GetSELinuxSupport(pathname string) (bool, error) {
	return false, nil
}


// GetMode returns permissions of the path.
func (hu *HostUtil) GetMode(pathname string) (os.FileMode, error) {
	info, err := os.Stat(pathname)
	if err != nil {
		return 0, err
	}
	return info.Mode(), nil
}


// GetSELinuxMountContext returns value of -o context=XYZ mount option on
// given mount point.
func (hu *HostUtil) GetSELinuxMountContext(pathname string) (string, error) {
	return "", nil
}


func NewIOHandler() IoUtil

// NewIOHandler Create a new IoHandler implementation
func NewIOHandler() IoUtil {
	return &osIOHandler{}
}


func (handler *osIOHandler) ReadFile(filename string) ([]byte, error) {
	return ioutil.ReadFile(filename)
}


func (handler *osIOHandler) ReadDir(dirname string) ([]os.FileInfo, error) {
	return ioutil.ReadDir(dirname)
}


func (handler *osIOHandler) Lstat(name string) (os.FileInfo, error) {
	return os.Lstat(name)
}


func (handler *osIOHandler) EvalSymlinks(path string) (string, error) {
	return filepath.EvalSymlinks(path)
}


func registerMetrics()

func registerMetrics() {
	// legacyregistry is the internal k8s wrapper around the prometheus
	// global registry, used specifically for metric stability enforcement
	legacyregistry.MustRegister(StorageOperationMetric)
	legacyregistry.MustRegister(storageOperationEndToEndLatencyMetric)
	legacyregistry.MustRegister(csiOperationsLatencyMetric)
}


// OperationCompleteHook returns a hook to call when an operation is completed
func OperationCompleteHook(plugin, operationName string) func(types.CompleteFuncParam) {
	requestTime := time.Now()
	opComplete := func(c types.CompleteFuncParam) {
		timeTaken := time.Since(requestTime).Seconds()
		// Create metric with operation name and plugin name
		status := statusSuccess
		if *c.Err != nil {
			// TODO: Establish well-known error codes to be able to distinguish
			// user configuration errors from system errors.
			status = statusFailUnknown
		}
		migrated := false
		if c.Migrated != nil {
			migrated = *c.Migrated
		}
		StorageOperationMetric.WithLabelValues(plugin, operationName, status, strconv.FormatBool(migrated)).Observe(timeTaken)
	}
	return opComplete
}


// FSGroupCompleteHook returns a hook to call when volume recursive permission is changed
func FSGroupCompleteHook(plugin volume.VolumePlugin, spec *volume.Spec) func(types.CompleteFuncParam) {
	return OperationCompleteHook(GetFullQualifiedPluginNameForVolume(plugin.GetPluginName(), spec), "volume_apply_access_control")
}


// GetFullQualifiedPluginNameForVolume returns full qualified plugin name for
// given volume. For CSI plugin, it appends plugin driver name at the end of
// plugin name, e.g. kubernetes.io/csi:csi-hostpath. It helps to distinguish
// between metrics emitted for CSI volumes which may be handled by different
// CSI plugin drivers.
func GetFullQualifiedPluginNameForVolume(pluginName string, spec *volume.Spec) string {
	if spec != nil {
		if spec.Volume != nil && spec.Volume.CSI != nil {
			return fmt.Sprintf("%s:%s", pluginName, spec.Volume.CSI.Driver)
		}
		if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.CSI != nil {
			return fmt.Sprintf("%s:%s", pluginName, spec.PersistentVolume.Spec.CSI.Driver)
		}
	}
	return pluginName
}


// RecordOperationLatencyMetric records the end to end latency for certain operation
// into metric volume_operation_total_seconds
func RecordOperationLatencyMetric(plugin, operationName string, secondsTaken float64) {
	storageOperationEndToEndLatencyMetric.WithLabelValues(plugin, operationName).Observe(secondsTaken)
}


// RecordCSIOperationLatencyMetrics records the CSI operation latency and grpc status
// into metric csi_kubelet_operations_seconds
func RecordCSIOperationLatencyMetrics(driverName string,
	operationName string,
	operationErr error,
	operationDuration time.Duration,
	migrated string) {
	csiOperationsLatencyMetric.WithLabelValues(driverName, operationName, getErrorCode(operationErr), migrated).Observe(operationDuration.Seconds())
}


func getErrorCode(err error) string

func getErrorCode(err error) string {
	if err == nil {
		return codes.OK.String()
	}

	st, ok := status.FromError(err)
	if !ok {
		// This is not gRPC error. The operation must have failed before gRPC
		// method was called, otherwise we would get gRPC error.
		return "unknown-non-grpc"
	}

	return st.Code().String()
}


// getNestedMountpoints returns a list of mountpoint directories that should be created
// for the volume indicated by name.
// note: the returned list is relative to baseDir
func getNestedMountpoints(name, baseDir string, pod v1.Pod) ([]string, error) {
	var retval []string
	checkContainer := func(container *v1.Container) error {
		var allMountPoints []string // all mount points in this container
		var myMountPoints []string  // mount points that match name
		for _, vol := range container.VolumeMounts {
			cleaned := filepath.Clean(vol.MountPath)
			allMountPoints = append(allMountPoints, cleaned)
			if vol.Name == name {
				myMountPoints = append(myMountPoints, cleaned)
			}
		}
		sort.Strings(allMountPoints)
		parentPrefix := ".." + string(os.PathSeparator)
		// Examine each place where this volume is mounted
		for _, myMountPoint := range myMountPoints {
			if strings.HasPrefix(myMountPoint, parentPrefix) {
				// Don't let a container trick us into creating directories outside of its rootfs
				return fmt.Errorf("invalid container mount point %v", myMountPoint)
			}
			myMPSlash := myMountPoint + string(os.PathSeparator)
			// The previously found nested mountpoints.
			// NOTE: We can't simply rely on sort.Strings to have all the mountpoints sorted and
			// grouped. For example, the following strings are sorted in this exact order:
			// /dir/nested, /dir/nested-vol, /dir/nested.vol, /dir/nested/double, /dir/nested2
			// The issue is a bit worse for Windows paths, since the \'s value is higher than /'s:
			// \dir\nested, \dir\nested-vol, \dir\nested.vol, \dir\nested2, \dir\nested\double
			// Because of this, we should use a list of previously mounted mountpoints, rather than only one.
			prevNestedMPs := []string{}
			// examine each mount point to see if it's nested beneath this volume
			// (but skip any that are double-nested beneath this volume)
			// For example, if this volume is mounted as /dir and other volumes are mounted
			//              as /dir/nested and /dir/nested/other, only create /dir/nested.
			for _, mp := range allMountPoints {
				if !strings.HasPrefix(mp, myMPSlash) {
					continue // skip -- not nested beneath myMountPoint
				}

				isNested := false
				for _, prevNestedMP := range prevNestedMPs {
					if strings.HasPrefix(mp, prevNestedMP) {
						isNested = true
						break
					}
				}
				if isNested {
					continue // skip -- double nested beneath myMountPoint
				}
				// since this mount point is nested, remember it so that we can check that following ones aren't nested beneath this one
				prevNestedMPs = append(prevNestedMPs, mp+string(os.PathSeparator))
				retval = append(retval, mp[len(myMPSlash):])
			}
		}
		return nil
	}

	var retErr error
	podutil.VisitContainers(&pod.Spec, podutil.AllFeatureEnabledContainers(), func(c *v1.Container, containerType podutil.ContainerType) bool {
		retErr = checkContainer(c)
		return retErr == nil
	})
	if retErr != nil {
		return nil, retErr
	}

	return retval, nil
}


// MakeNestedMountpoints creates mount points in baseDir for volumes mounted beneath name
func MakeNestedMountpoints(name, baseDir string, pod v1.Pod) error {
	dirs, err := getNestedMountpoints(name, baseDir, pod)
	if err != nil {
		return err
	}
	for _, dir := range dirs {
		err := os.MkdirAll(filepath.Join(baseDir, dir), 0755)
		if err != nil {
			return fmt.Errorf("unable to create nested volume mountpoints: %v", err)
		}
	}
	return nil
}


// NewNestedPendingOperations returns a new instance of NestedPendingOperations.
func NewNestedPendingOperations(exponentialBackOffOnError bool) NestedPendingOperations {
	g := &nestedPendingOperations{
		operations:                []operation{},
		exponentialBackOffOnError: exponentialBackOffOnError,
	}
	g.cond = sync.NewCond(&g.lock)
	return g
}


func (grm *nestedPendingOperations) Run(
	volumeName v1.UniqueVolumeName,
	podName volumetypes.UniquePodName,
	nodeName types.NodeName,
	generatedOperations volumetypes.GeneratedOperations) error {
	grm.lock.Lock()
	defer grm.lock.Unlock()

	opKey := operationKey{volumeName, podName, nodeName}

	opExists, previousOpIndex := grm.isOperationExists(opKey)
	if opExists {
		previousOp := grm.operations[previousOpIndex]
		// Operation already exists
		if previousOp.operationPending {
			// Operation is pending
			return NewAlreadyExistsError(opKey)
		}

		backOffErr := previousOp.expBackoff.SafeToRetry(fmt.Sprintf("%+v", opKey))
		if backOffErr != nil {
			if previousOp.operationName == generatedOperations.OperationName {
				return backOffErr
			}
			// previous operation and new operation are different. reset op. name and exp. backoff
			grm.operations[previousOpIndex].operationName = generatedOperations.OperationName
			grm.operations[previousOpIndex].expBackoff = exponentialbackoff.ExponentialBackoff{}
		}

		// Update existing operation to mark as pending.
		grm.operations[previousOpIndex].operationPending = true
		grm.operations[previousOpIndex].key = opKey
	} else {
		// Create a new operation
		grm.operations = append(grm.operations,
			operation{
				key:              opKey,
				operationPending: true,
				operationName:    generatedOperations.OperationName,
				expBackoff:       exponentialbackoff.ExponentialBackoff{},
			})
	}

	go func() (eventErr, detailedErr error) {
		// Handle unhandled panics (very unlikely)
		defer k8sRuntime.HandleCrash()
		// Handle completion of and error, if any, from operationFunc()
		defer grm.operationComplete(opKey, &detailedErr)
		return generatedOperations.Run()
	}()

	return nil
}


func (grm *nestedPendingOperations) IsOperationSafeToRetry(
	volumeName v1.UniqueVolumeName,
	podName volumetypes.UniquePodName,
	nodeName types.NodeName,
	operationName string) bool {

	grm.lock.RLock()
	defer grm.lock.RUnlock()

	opKey := operationKey{volumeName, podName, nodeName}
	exist, previousOpIndex := grm.isOperationExists(opKey)
	if !exist {
		return true
	}
	previousOp := grm.operations[previousOpIndex]
	if previousOp.operationPending {
		return false
	}
	backOffErr := previousOp.expBackoff.SafeToRetry(fmt.Sprintf("%+v", opKey))
	if backOffErr != nil {
		if previousOp.operationName == operationName {
			return false
		}
	}

	return true
}


func (grm *nestedPendingOperations) IsOperationPending(
	volumeName v1.UniqueVolumeName,
	podName volumetypes.UniquePodName,
	nodeName types.NodeName) bool {

	grm.lock.RLock()
	defer grm.lock.RUnlock()

	opKey := operationKey{volumeName, podName, nodeName}
	exist, previousOpIndex := grm.isOperationExists(opKey)
	if exist && grm.operations[previousOpIndex].operationPending {
		return true
	}
	return false
}


// This is an internal function and caller should acquire and release the lock
func (grm *nestedPendingOperations) isOperationExists(key operationKey) (bool, int) {

	// If volumeName is empty, operation can be executed concurrently
	if key.volumeName == EmptyUniqueVolumeName {
		return false, -1
	}

	opIndex := -1
	for previousOpIndex, previousOp := range grm.operations {
		volumeNameMatch := previousOp.key.volumeName == key.volumeName

		podNameMatch := previousOp.key.podName == EmptyUniquePodName ||
			key.podName == EmptyUniquePodName ||
			previousOp.key.podName == key.podName

		podNameExactMatch := previousOp.key.podName == key.podName

		nodeNameMatch := previousOp.key.nodeName == EmptyNodeName ||
			key.nodeName == EmptyNodeName ||
			previousOp.key.nodeName == key.nodeName

		nodeNameExactMatch := previousOp.key.nodeName == key.nodeName

		if volumeNameMatch && podNameMatch && nodeNameMatch {
			// nonExactMatch pending first
			if previousOp.operationPending {
				return true, previousOpIndex
			}
			// nonExactMatch with no pending, set opIndex to the first nonExactMatch
			// exactMatch can override opIndex to expected
			if opIndex == -1 || (podNameExactMatch && nodeNameExactMatch) {
				opIndex = previousOpIndex
			}
		}
	}
	return opIndex != -1, opIndex

}


func (grm *nestedPendingOperations) getOperation(key operationKey) (uint, error) {
	// Assumes lock has been acquired by caller.

	for i, op := range grm.operations {
		if op.key.volumeName == key.volumeName &&
			op.key.podName == key.podName &&
			op.key.nodeName == key.nodeName {
			return uint(i), nil
		}
	}

	return 0, fmt.Errorf("operation %+v not found", key)
}


func (grm *nestedPendingOperations) deleteOperation(key operationKey) {
	// Assumes lock has been acquired by caller.

	opIndex := -1
	for i, op := range grm.operations {
		if op.key.volumeName == key.volumeName &&
			op.key.podName == key.podName &&
			op.key.nodeName == key.nodeName {
			opIndex = i
			break
		}
	}

	if opIndex < 0 {
		return
	}

	// Delete index without preserving order
	grm.operations[opIndex] = grm.operations[len(grm.operations)-1]
	grm.operations = grm.operations[:len(grm.operations)-1]
}


func (grm *nestedPendingOperations) operationComplete(key operationKey, err *error) {
	// Defer operations are executed in Last-In is First-Out order. In this case
	// the lock is acquired first when operationCompletes begins, and is
	// released when the method finishes, after the lock is released cond is
	// signaled to wake waiting goroutine.
	defer grm.cond.Signal()
	grm.lock.Lock()
	defer grm.lock.Unlock()

	if *err == nil || !grm.exponentialBackOffOnError {
		// Operation completed without error, or exponentialBackOffOnError disabled
		grm.deleteOperation(key)
		if *err != nil {
			// Log error
			klog.Errorf("operation %+v failed with: %v", key, *err)
		}
		return
	}

	// Operation completed with error and exponentialBackOffOnError Enabled
	existingOpIndex, getOpErr := grm.getOperation(key)
	if getOpErr != nil {
		// Failed to find existing operation
		klog.Errorf("Operation %+v completed. error: %v. exponentialBackOffOnError is enabled, but failed to get operation to update.",
			key,
			*err)
		return
	}

	grm.operations[existingOpIndex].expBackoff.Update(err)
	grm.operations[existingOpIndex].operationPending = false

	// Log error
	klog.Errorf("%v", grm.operations[existingOpIndex].expBackoff.
		GenerateNoRetriesPermittedMsg(fmt.Sprintf("%+v", key)))
}


func (grm *nestedPendingOperations) Wait() {
	grm.lock.Lock()
	defer grm.lock.Unlock()

	for len(grm.operations) > 0 {
		grm.cond.Wait()
	}
}


// NewAlreadyExistsError returns a new instance of AlreadyExists error.
func NewAlreadyExistsError(key operationKey) error {
	return alreadyExistsError{key}
}


func IsAlreadyExists(err error) bool

// IsAlreadyExists returns true if an error returned from
// NestedPendingOperations indicates a new operation can not be started because
// an operation with the same operation name is already executing.
func IsAlreadyExists(err error) bool {
	switch err.(type) {
	case alreadyExistsError:
		return true
	default:
		return false
	}
}


func (err alreadyExistsError) Error() string {
	return fmt.Sprintf(
		"Failed to create operation with name %+v. An operation with that name is already executing.",
		err.operationKey)
}


func newNodeExpander(resizeOp nodeResizeOperationOpts, client clientset.Interface, recorder record.EventRecorder) *NodeExpander {
	return &NodeExpander{
		kubeClient:              client,
		nodeResizeOperationOpts: resizeOp,
		recorder:                recorder,
	}
}


// runPreCheck performs some sanity checks before expansion can be performed on the PVC.
func (ne *NodeExpander) runPreCheck() bool {
	ne.pvcStatusCap = ne.pvc.Status.Capacity[v1.ResourceStorage]
	ne.pvCap = ne.pv.Spec.Capacity[v1.ResourceStorage]

	ne.resizeStatus = ne.pvc.Status.ResizeStatus

	// PVC is already expanded but we are still trying to expand the volume because
	// last recorded size in ASOW is older. This can happen for RWX volume types.
	if ne.pvcStatusCap.Cmp(ne.pluginResizeOpts.NewSize) >= 0 && (ne.resizeStatus == nil || *ne.resizeStatus == v1.PersistentVolumeClaimNoExpansionInProgress) {
		ne.pvcAlreadyUpdated = true
	}

	// if resizestatus is nil or NodeExpansionInProgress or NodeExpansionPending then we
	// should allow volume expansion on the node to proceed. We are making an exception for
	// resizeStatus being nil because it will support use cases where
	// resizeStatus may not be set (old control-plane expansion controller etc).
	if ne.resizeStatus == nil ||
		ne.pvcAlreadyUpdated ||
		*ne.resizeStatus == v1.PersistentVolumeClaimNodeExpansionPending ||
		*ne.resizeStatus == v1.PersistentVolumeClaimNodeExpansionInProgress {
		return true
	}

	return false
}


func (ne *NodeExpander) expandOnPlugin() (bool, error, testResponseData) {
	allowExpansion := ne.runPreCheck()
	if !allowExpansion {
		return false, nil, testResponseData{false, true}
	}

	var err error
	nodeName := ne.vmt.Pod.Spec.NodeName

	if !ne.pvcAlreadyUpdated {
		ne.pvc, err = util.MarkNodeExpansionInProgress(ne.pvc, ne.kubeClient)

		if err != nil {
			msg := ne.vmt.GenerateErrorDetailed("MountVolume.NodeExpandVolume failed to mark node expansion in progress: %v", err)
			klog.Errorf(msg.Error())
			return false, err, testResponseData{}
		}
	}
	_, resizeErr := ne.volumePlugin.NodeExpand(ne.pluginResizeOpts)
	if resizeErr != nil {
		if volumetypes.IsOperationFinishedError(resizeErr) {
			var markFailedError error
			ne.pvc, markFailedError = util.MarkNodeExpansionFailed(ne.pvc, ne.kubeClient)
			if markFailedError != nil {
				klog.Errorf(ne.vmt.GenerateErrorDetailed("MountMount.NodeExpandVolume failed to mark node expansion as failed: %v", err).Error())
			}
		}

		// if driver returned FailedPrecondition error that means
		// volume expansion should not be retried on this node but
		// expansion operation should not block mounting
		if volumetypes.IsFailedPreconditionError(resizeErr) {
			ne.actualStateOfWorld.MarkForInUseExpansionError(ne.vmt.VolumeName)
			klog.Errorf(ne.vmt.GenerateErrorDetailed("MountVolume.NodeExapndVolume failed with %v", resizeErr).Error())
			return false, nil, testResponseData{assumeResizeFinished: true, resizeCalledOnPlugin: true}
		}
		return false, resizeErr, testResponseData{assumeResizeFinished: true, resizeCalledOnPlugin: true}
	}
	simpleMsg, detailedMsg := ne.vmt.GenerateMsg("MountVolume.NodeExpandVolume succeeded", nodeName)
	ne.recorder.Eventf(ne.vmt.Pod, v1.EventTypeNormal, kevents.FileSystemResizeSuccess, simpleMsg)
	ne.recorder.Eventf(ne.pvc, v1.EventTypeNormal, kevents.FileSystemResizeSuccess, simpleMsg)
	klog.InfoS(detailedMsg, "pod", klog.KObj(ne.vmt.Pod))

	// no need to update PVC object if we already updated it
	if ne.pvcAlreadyUpdated {
		return true, nil, testResponseData{true, true}
	}

	// File system resize succeeded, now update the PVC's Capacity to match the PV's
	ne.pvc, err = util.MarkFSResizeFinished(ne.pvc, ne.pluginResizeOpts.NewSize, ne.kubeClient)
	if err != nil {
		return true, fmt.Errorf("mountVolume.NodeExpandVolume update pvc status failed: %v", err), testResponseData{true, true}
	}
	return true, nil, testResponseData{true, true}
}


func NewOperationExecutor(

// NewOperationExecutor returns a new instance of OperationExecutor.
func NewOperationExecutor(
	operationGenerator OperationGenerator) OperationExecutor {

	return &operationExecutor{
		pendingOperations: nestedpendingoperations.NewNestedPendingOperations(
			true /* exponentialBackOffOnError */),
		operationGenerator: operationGenerator,
	}
}


func errSuffix(err error) string

// Generates an error string with the format ": <err>" if err exists
func errSuffix(err error) string {
	errStr := ""
	if err != nil {
		errStr = fmt.Sprintf(": %v", err)
	}
	return errStr
}


// Generate a detailed error msg for logs
func generateVolumeMsgDetailed(prefixMsg, suffixMsg, volumeName, details string) (detailedMsg string) {
	return fmt.Sprintf("%v for volume %q %v %v", prefixMsg, volumeName, details, suffixMsg)
}


// Generate a simplified error msg for events and a detailed error msg for logs
func generateVolumeMsg(prefixMsg, suffixMsg, volumeName, details string) (simpleMsg, detailedMsg string) {
	simpleMsg = fmt.Sprintf("%v for volume %q %v", prefixMsg, volumeName, suffixMsg)
	return simpleMsg, generateVolumeMsgDetailed(prefixMsg, suffixMsg, volumeName, details)
}


// GenerateMsgDetailed returns detailed msgs for volumes to attach
func (volume *VolumeToAttach) GenerateMsgDetailed(prefixMsg, suffixMsg string) (detailedMsg string) {
	detailedStr := fmt.Sprintf("(UniqueName: %q) from node %q", volume.VolumeName, volume.NodeName)
	volumeSpecName := "nil"
	if volume.VolumeSpec != nil {
		volumeSpecName = volume.VolumeSpec.Name()
	}
	return generateVolumeMsgDetailed(prefixMsg, suffixMsg, volumeSpecName, detailedStr)
}


// GenerateMsg returns simple and detailed msgs for volumes to attach
func (volume *VolumeToAttach) GenerateMsg(prefixMsg, suffixMsg string) (simpleMsg, detailedMsg string) {
	detailedStr := fmt.Sprintf("(UniqueName: %q) from node %q", volume.VolumeName, volume.NodeName)
	volumeSpecName := "nil"
	if volume.VolumeSpec != nil {
		volumeSpecName = volume.VolumeSpec.Name()
	}
	return generateVolumeMsg(prefixMsg, suffixMsg, volumeSpecName, detailedStr)
}


// GenerateErrorDetailed returns detailed errors for volumes to attach
func (volume *VolumeToAttach) GenerateErrorDetailed(prefixMsg string, err error) (detailedErr error) {
	return fmt.Errorf(volume.GenerateMsgDetailed(prefixMsg, errSuffix(err)))
}


// GenerateError returns simple and detailed errors for volumes to attach
func (volume *VolumeToAttach) GenerateError(prefixMsg string, err error) (simpleErr, detailedErr error) {
	simpleMsg, detailedMsg := volume.GenerateMsg(prefixMsg, errSuffix(err))
	return fmt.Errorf(simpleMsg), fmt.Errorf(detailedMsg)
}


// String combines key fields of the volume for logging in text format.
func (volume *VolumeToAttach) String() string {
	volumeSpecName := "nil"
	if volume.VolumeSpec != nil {
		volumeSpecName = volume.VolumeSpec.Name()
	}
	return fmt.Sprintf("%s (UniqueName: %s) from node %s", volumeSpecName, volume.VolumeName, volume.NodeName)
}