GetInodes

// GetInodes returns the inodes in use if available via quotas
func (v linuxVolumeQuotaApplier) GetInodes(_ string, id QuotaID) (int64, error) {
	return getQuantity(v.mountpoint, id, "-i", 1, v.allowEmptyOutput)
}


QuotaIDIsInUse

// QuotaIDIsInUse checks whether the specified quota ID is in use on the specified
// filesystem
func (v linuxVolumeQuotaApplier) QuotaIDIsInUse(id QuotaID) (bool, error) {
	bytes, err := v.GetConsumption(v.mountpoint, id)
	if err != nil {
		return false, err
	}
	if bytes > 0 {
		return true, nil
	}
	inodes, err := v.GetInodes(v.mountpoint, id)
	return inodes > 0, err
}


projFilesAreOK

func projFilesAreOK() error {
	if sf, err := os.Lstat(projectsFile); err != nil || sf.Mode().IsRegular() {
		if sf, err := os.Lstat(projidFile); err != nil || sf.Mode().IsRegular() {
			return nil
		}
		return fmt.Errorf("%s exists but is not a plain file, cannot continue", projidFile)
	}
	return fmt.Errorf("%s exists but is not a plain file, cannot continue", projectsFile)
}


lockFile

func lockFile(file *os.File) error {
	return unix.Flock(int(file.Fd()), unix.LOCK_EX)
}


unlockFile

func unlockFile(file *os.File) error {
	return unix.Flock(int(file.Fd()), unix.LOCK_UN)
}


openAndLockProjectFiles

// openAndLockProjectFiles opens /etc/projects and /etc/projid locked.
// Creates them if they don't exist
func openAndLockProjectFiles() (*os.File, *os.File, error) {
	// Make sure neither project-related file is a symlink!
	if err := projFilesAreOK(); err != nil {
		return nil, nil, fmt.Errorf("system project files failed verification: %v", err)
	}
	// We don't actually modify the original files; we create temporaries and
	// move them over the originals
	fProjects, err := os.OpenFile(projectsFile, os.O_RDONLY|os.O_CREATE, 0644)
	if err != nil {
		err = fmt.Errorf("unable to open %s: %v", projectsFile, err)
		return nil, nil, err
	}
	fProjid, err := os.OpenFile(projidFile, os.O_RDONLY|os.O_CREATE, 0644)
	if err == nil {
		// Check once more, to ensure nothing got changed out from under us
		if err = projFilesAreOK(); err == nil {
			err = lockFile(fProjects)
			if err == nil {
				err = lockFile(fProjid)
				if err == nil {
					return fProjects, fProjid, nil
				}
				// Nothing useful we can do if we get an error here
				err = fmt.Errorf("unable to lock %s: %v", projidFile, err)
				unlockFile(fProjects)
			} else {
				err = fmt.Errorf("unable to lock %s: %v", projectsFile, err)
			}
		} else {
			err = fmt.Errorf("system project files failed re-verification: %v", err)
		}
		fProjid.Close()
	} else {
		err = fmt.Errorf("unable to open %s: %v", projidFile, err)
	}
	fProjects.Close()
	return nil, nil, err
}


closeProjectFiles

func closeProjectFiles(fProjects *os.File, fProjid *os.File) error {
	// Nothing useful we can do if either of these fail,
	// but we have to close (and thereby unlock) the files anyway.
	var err error
	var err1 error
	if fProjid != nil {
		err = fProjid.Close()
	}
	if fProjects != nil {
		err1 = fProjects.Close()
	}
	if err == nil {
		return err1
	}
	return err
}


parseProject

func parseProject(l string) projectType {
	if match := projectsParseRegexp.FindStringSubmatch(l); match != nil {
		i, err := strconv.Atoi(match[1])
		if err == nil {
			return projectType{true, common.QuotaID(i), match[2], l}
		}
	}
	return projectType{true, common.BadQuotaID, "", l}
}


parseProjid

func parseProjid(l string) projectType {
	if match := projidParseRegexp.FindStringSubmatch(l); match != nil {
		i, err := strconv.Atoi(match[2])
		if err == nil {
			return projectType{true, common.QuotaID(i), match[1], l}
		}
	}
	return projectType{true, common.BadQuotaID, "", l}
}


parseProjFile

func parseProjFile(f *os.File, parser func(l string) projectType) []projectType {
	var answer []projectType
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		answer = append(answer, parser(scanner.Text()))
	}
	return answer
}


readProjectFiles

func readProjectFiles(projects *os.File, projid *os.File) projectsList {
	return projectsList{parseProjFile(projects, parseProject), parseProjFile(projid, parseProjid)}
}


findAvailableQuota

// findAvailableQuota finds the next available quota from the FirstQuota
// it returns error if QuotaIDIsInUse returns error when getting quota id in use;
// it searches at most maxUnusedQuotasToSearch(128) time
func findAvailableQuota(path string, idMap map[common.QuotaID]bool) (common.QuotaID, error) {
	unusedQuotasSearched := 0
	for id := common.FirstQuota; true; id++ {
		if _, ok := idMap[id]; !ok {
			isInUse, err := getApplier(path).QuotaIDIsInUse(id)
			if err != nil {
				return common.BadQuotaID, err
			} else if !isInUse {
				return id, nil
			}
			unusedQuotasSearched++
			if unusedQuotasSearched > maxUnusedQuotasToSearch {
				break
			}
		}
	}
	return common.BadQuotaID, fmt.Errorf("cannot find available quota ID")
}


addDirToProject

func addDirToProject(path string, id common.QuotaID, list *projectsList) (common.QuotaID, bool, error) {
	idMap := make(map[common.QuotaID]bool)
	for _, project := range list.projects {
		if project.data == path {
			if id != common.BadQuotaID && id != project.id {
				return common.BadQuotaID, false, fmt.Errorf("attempt to reassign project ID for %s", path)
			}
			// Trying to reassign a directory to the project it's
			// already in.  Maybe this should be an error, but for
			// now treat it as an idempotent operation
			return project.id, false, nil
		}
		idMap[project.id] = true
	}
	var needToAddProjid = true
	for _, projid := range list.projid {
		idMap[projid.id] = true
		if projid.id == id && id != common.BadQuotaID {
			needToAddProjid = false
		}
	}
	var err error
	if id == common.BadQuotaID {
		id, err = findAvailableQuota(path, idMap)
		if err != nil {
			return common.BadQuotaID, false, err
		}
		needToAddProjid = true
	}
	if needToAddProjid {
		name := fmt.Sprintf("volume%v", id)
		line := fmt.Sprintf("%s:%v", name, id)
		list.projid = append(list.projid, projectType{true, id, name, line})
	}
	line := fmt.Sprintf("%v:%s", id, path)
	list.projects = append(list.projects, projectType{true, id, path, line})
	return id, needToAddProjid, nil
}


removeDirFromProject

func removeDirFromProject(path string, id common.QuotaID, list *projectsList) (bool, error) {
	if id == common.BadQuotaID {
		return false, fmt.Errorf("attempt to remove invalid quota ID from %s", path)
	}
	foundAt := -1
	countByID := make(map[common.QuotaID]int)
	for i, project := range list.projects {
		if project.data == path {
			if id != project.id {
				return false, fmt.Errorf("attempting to remove quota ID %v from path %s, but expecting ID %v", id, path, project.id)
			} else if foundAt != -1 {
				return false, fmt.Errorf("found multiple quota IDs for path %s", path)
			}
			// Faster and easier than deleting an element
			list.projects[i].isValid = false
			foundAt = i
		}
		countByID[project.id]++
	}
	if foundAt == -1 {
		return false, fmt.Errorf("cannot find quota associated with path %s", path)
	}
	if countByID[id] <= 1 {
		// Removing the last entry means that we're no longer using
		// the quota ID, so remove that as well
		for i, projid := range list.projid {
			if projid.id == id {
				list.projid[i].isValid = false
			}
		}
		return true, nil
	}
	return false, nil
}


writeProjectFile

func writeProjectFile(base *os.File, projects []projectType) (string, error) {
	oname := base.Name()
	stat, err := base.Stat()
	if err != nil {
		return "", err
	}
	mode := stat.Mode() & os.ModePerm
	f, err := ioutil.TempFile(filepath.Dir(oname), filepath.Base(oname))
	if err != nil {
		return "", err
	}
	filename := f.Name()
	if err := os.Chmod(filename, mode); err != nil {
		return "", err
	}
	for _, proj := range projects {
		if proj.isValid {
			if _, err := f.WriteString(fmt.Sprintf("%s\n", proj.line)); err != nil {
				f.Close()
				os.Remove(filename)
				return "", err
			}
		}
	}
	if err := f.Close(); err != nil {
		os.Remove(filename)
		return "", err
	}
	return filename, nil
}


writeProjectFiles

func writeProjectFiles(fProjects *os.File, fProjid *os.File, writeProjid bool, list projectsList) error {
	tmpProjects, err := writeProjectFile(fProjects, list.projects)
	if err == nil {
		// Ensure that both files are written before we try to rename either.
		if writeProjid {
			tmpProjid, err := writeProjectFile(fProjid, list.projid)
			if err == nil {
				err = os.Rename(tmpProjid, fProjid.Name())
				if err != nil {
					os.Remove(tmpProjid)
				}
			}
		}
		if err == nil {
			err = os.Rename(tmpProjects, fProjects.Name())
			if err == nil {
				return nil
			}
			// We're in a bit of trouble here; at this
			// point we've successfully renamed tmpProjid
			// to the real thing, but renaming tmpProject
			// to the real file failed.  There's not much we
			// can do in this position.  Anything we could do
			// to try to undo it would itself be likely to fail.
		}
		os.Remove(tmpProjects)
	}
	return fmt.Errorf("unable to write project files: %v", err)
}


createProjectID

// if ID is common.BadQuotaID, generate new project id if the dir is not in a project
func createProjectID(path string, ID common.QuotaID) (common.QuotaID, error) {
	quotaIDLock.Lock()
	defer quotaIDLock.Unlock()
	fProjects, fProjid, err := openAndLockProjectFiles()
	if err == nil {
		defer closeProjectFiles(fProjects, fProjid)
		list := readProjectFiles(fProjects, fProjid)
		var writeProjid bool
		ID, writeProjid, err = addDirToProject(path, ID, &list)
		if err == nil && ID != common.BadQuotaID {
			if err = writeProjectFiles(fProjects, fProjid, writeProjid, list); err == nil {
				return ID, nil
			}
		}
	}
	return common.BadQuotaID, fmt.Errorf("createProjectID %s %v failed %v", path, ID, err)
}


removeProjectID

func removeProjectID(path string, ID common.QuotaID) error {
	if ID == common.BadQuotaID {
		return fmt.Errorf("attempting to remove invalid quota ID %v", ID)
	}
	quotaIDLock.Lock()
	defer quotaIDLock.Unlock()
	fProjects, fProjid, err := openAndLockProjectFiles()
	if err == nil {
		defer closeProjectFiles(fProjects, fProjid)
		list := readProjectFiles(fProjects, fProjid)
		var writeProjid bool
		writeProjid, err = removeDirFromProject(path, ID, &list)
		if err == nil {
			if err = writeProjectFiles(fProjects, fProjid, writeProjid, list); err == nil {
				return nil
			}
		}
	}
	return fmt.Errorf("removeProjectID %s %v failed %v", path, ID, err)
}


enabledQuotasForMonitoring

func enabledQuotasForMonitoring() bool {
	return utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolationFSQuotaMonitoring)
}


detectBackingDevInternal

// Separate the innards for ease of testing
func detectBackingDevInternal(mountpoint string, mounts string) (string, error) {
	file, err := os.Open(mounts)
	if err != nil {
		return "", err
	}
	defer file.Close()
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		match := common.MountParseRegexp.FindStringSubmatch(scanner.Text())
		if match != nil {
			device := match[1]
			mount := match[2]
			if mount == mountpoint {
				return device, nil
			}
		}
	}
	return "", fmt.Errorf("couldn't find backing device for %s", mountpoint)
}


detectBackingDev

// detectBackingDev assumes that the mount point provided is valid
func detectBackingDev(_ mount.Interface, mountpoint string) (string, error) {
	return detectBackingDevInternal(mountpoint, common.MountsFile)
}


clearBackingDev

func clearBackingDev(path string) {
	backingDevLock.Lock()
	defer backingDevLock.Unlock()
	delete(backingDevMap, path)
}


detectMountpointInternal

// Assumes that the path has been fully canonicalized
// Breaking this up helps with testing
func detectMountpointInternal(m mount.Interface, path string) (string, error) {
	for path != "" && path != "/" {
		// per k8s.io/mount-utils/mount_linux this detects all but
		// a bind mount from one part of a mount to another.
		// For our purposes that's fine; we simply want the "true"
		// mount point
		//
		// IsNotMountPoint proved much more troublesome; it actually
		// scans the mounts, and when a lot of mount/unmount
		// activity takes place, it is not able to get a consistent
		// view of /proc/self/mounts, causing it to time out and
		// report incorrectly.
		isNotMount, err := m.IsLikelyNotMountPoint(path)
		if err != nil {
			return "/", err
		}
		if !isNotMount {
			return path, nil
		}
		path = filepath.Dir(path)
	}
	return "/", nil
}


detectMountpoint

func detectMountpoint(m mount.Interface, path string) (string, error) {
	xpath, err := filepath.Abs(path)
	if err != nil {
		return "/", err
	}
	xpath, err = filepath.EvalSymlinks(xpath)
	if err != nil {
		return "/", err
	}
	if xpath, err = detectMountpointInternal(m, xpath); err == nil {
		return xpath, nil
	}
	return "/", err
}


clearMountpoint

func clearMountpoint(path string) {
	mountpointLock.Lock()
	defer mountpointLock.Unlock()
	delete(mountpointMap, path)
}


getFSInfo

// getFSInfo Returns mountpoint and backing device
// getFSInfo should cache the mountpoint and backing device for the
// path.
func getFSInfo(m mount.Interface, path string) (string, string, error) {
	mountpointLock.Lock()
	defer mountpointLock.Unlock()

	backingDevLock.Lock()
	defer backingDevLock.Unlock()

	var err error

	mountpoint, okMountpoint := mountpointMap[path]
	if !okMountpoint {
		mountpoint, err = detectMountpoint(m, path)
		if err != nil {
			return "", "", fmt.Errorf("cannot determine mountpoint for %s: %v", path, err)
		}
	}

	backingDev, okBackingDev := backingDevMap[path]
	if !okBackingDev {
		backingDev, err = detectBackingDev(m, mountpoint)
		if err != nil {
			return "", "", fmt.Errorf("cannot determine backing device for %s: %v", path, err)
		}
	}
	mountpointMap[path] = mountpoint
	backingDevMap[path] = backingDev
	return mountpoint, backingDev, nil
}


clearFSInfo

func clearFSInfo(path string) {
	clearMountpoint(path)
	clearBackingDev(path)
}


getApplier

func getApplier(path string) common.LinuxVolumeQuotaApplier {
	dirApplierLock.Lock()
	defer dirApplierLock.Unlock()
	return dirApplierMap[path]
}


setApplier

func setApplier(path string, applier common.LinuxVolumeQuotaApplier) {
	dirApplierLock.Lock()
	defer dirApplierLock.Unlock()
	dirApplierMap[path] = applier
}


clearApplier

func clearApplier(path string) {
	dirApplierLock.Lock()
	defer dirApplierLock.Unlock()
	delete(dirApplierMap, path)
}


setQuotaOnDir

func setQuotaOnDir(path string, id common.QuotaID, bytes int64) error {
	return getApplier(path).SetQuotaOnDir(path, id, bytes)
}


GetQuotaOnDir

func GetQuotaOnDir(m mount.Interface, path string) (common.QuotaID, error) {
	_, _, err := getFSInfo(m, path)
	if err != nil {
		return common.BadQuotaID, err
	}
	return getApplier(path).GetQuotaOnDir(path)
}


clearQuotaOnDir

func clearQuotaOnDir(m mount.Interface, path string) error {
	// Since we may be called without path being in the map,
	// we explicitly have to check in this case.
	klog.V(4).Infof("clearQuotaOnDir %s", path)
	supportsQuotas, err := SupportsQuotas(m, path)
	if err != nil {
		// Log-and-continue instead of returning an error for now
		// due to unspecified backwards compatibility concerns (a subject to revise)
		klog.V(3).Infof("Attempt to check for quota support failed: %v", err)
	}
	if !supportsQuotas {
		return nil
	}
	projid, err := GetQuotaOnDir(m, path)
	if err == nil && projid != common.BadQuotaID {
		// This means that we have a quota on the directory but
		// we can't clear it.  That's not good.
		err = setQuotaOnDir(path, projid, 0)
		if err != nil {
			klog.V(3).Infof("Attempt to clear quota failed: %v", err)
		}
		// Even if clearing the quota failed, we still need to
		// try to remove the project ID, or that may be left dangling.
		err1 := removeProjectID(path, projid)
		if err1 != nil {
			klog.V(3).Infof("Attempt to remove quota ID from system files failed: %v", err1)
		}
		clearFSInfo(path)
		if err != nil {
			return err
		}
		return err1
	}
	// If we couldn't get a quota, that's fine -- there may
	// never have been one, and we have no way to know otherwise
	klog.V(3).Infof("clearQuotaOnDir fails %v", err)
	return nil
}


SupportsQuotas

// SupportsQuotas -- Does the path support quotas
// Cache the applier for paths that support quotas.  For paths that don't,
// don't cache the result because nothing will clean it up.
// However, do cache the device->applier map; the number of devices
// is bounded.
func SupportsQuotas(m mount.Interface, path string) (bool, error) {
	if !enabledQuotasForMonitoring() {
		klog.V(3).Info("SupportsQuotas called, but quotas disabled")
		return false, nil
	}
	supportsQuotasLock.Lock()
	defer supportsQuotasLock.Unlock()
	if supportsQuotas, ok := supportsQuotasMap[path]; ok {
		return supportsQuotas, nil
	}
	mount, dev, err := getFSInfo(m, path)
	if err != nil {
		return false, err
	}
	// Do we know about this device?
	applier, ok := devApplierMap[mount]
	if !ok {
		for _, provider := range providers {
			if applier = provider.GetQuotaApplier(mount, dev); applier != nil {
				devApplierMap[mount] = applier
				break
			}
		}
	}
	if applier != nil {
		supportsQuotasMap[path] = true
		setApplier(path, applier)
		return true, nil
	}
	delete(backingDevMap, path)
	delete(mountpointMap, path)
	return false, nil
}


AssignQuota

// AssignQuota -- assign a quota to the specified directory.
// AssignQuota chooses the quota ID based on the pod UID and path.
// If the pod UID is identical to another one known, it may (but presently
// doesn't) choose the same quota ID as other volumes in the pod.
func AssignQuota(m mount.Interface, path string, poduid types.UID, bytes *resource.Quantity) error { //nolint:staticcheck
	if bytes == nil {
		return fmt.Errorf("attempting to assign null quota to %s", path)
	}
	ibytes := bytes.Value()
	if ok, err := SupportsQuotas(m, path); !ok {
		return fmt.Errorf("quotas not supported on %s: %v", path, err)
	}
	quotaLock.Lock()
	defer quotaLock.Unlock()
	// Current policy is to set individual quotas on each volume,
	// for each new volume we generate a random UUID and we use that as
	// the internal pod uid.
	// From fsquota point of view each volume is attached to a
	// single unique pod.
	// If we decide later that we want to assign one quota for all
	// volumes in a pod, we can simply use poduid parameter directly
	// If and when we decide permanently that we're going to adopt
	// one quota per volume, we can rip all of the pod code out.
	externalPodUid := poduid
	internalPodUid, ok := dirPodMap[path]
	if ok {
		if podUidMap[internalPodUid] != externalPodUid {
			return fmt.Errorf("requesting quota on existing directory %s but different pod %s %s", path, podUidMap[internalPodUid], externalPodUid)
		}
	} else {
		internalPodUid = types.UID(uuid.NewUUID())
	}
	oid, ok := podQuotaMap[internalPodUid]
	if ok {
		if quotaSizeMap[oid] != ibytes {
			return fmt.Errorf("requesting quota of different size: old %v new %v", quotaSizeMap[oid], bytes)
		}
		if _, ok := dirPodMap[path]; ok {
			return nil
		}
	} else {
		oid = common.BadQuotaID
	}
	id, err := createProjectID(path, oid)
	if err == nil {
		if oid != common.BadQuotaID && oid != id {
			return fmt.Errorf("attempt to reassign quota %v to %v", oid, id)
		}
		// When enforcing quotas are enabled, we'll condition this
		// on their being disabled also.
		if ibytes > 0 {
			ibytes = -1
		}
		if err = setQuotaOnDir(path, id, ibytes); err == nil {
			quotaPodMap[id] = internalPodUid
			quotaSizeMap[id] = ibytes
			podQuotaMap[internalPodUid] = id
			dirQuotaMap[path] = id
			dirPodMap[path] = internalPodUid
			podUidMap[internalPodUid] = externalPodUid
			podDirCountMap[internalPodUid]++
			klog.V(4).Infof("Assigning quota ID %d (%d) to %s", id, ibytes, path)
			return nil
		}
		removeProjectID(path, id)
	}
	return fmt.Errorf("assign quota FAILED %v", err)
}


GetConsumption

// GetConsumption -- retrieve the consumption (in bytes) of the directory
func GetConsumption(path string) (*resource.Quantity, error) {
	// Note that we actually need to hold the lock at least through
	// running the quota command, so it can't get recycled behind our back
	quotaLock.Lock()
	defer quotaLock.Unlock()
	applier := getApplier(path)
	// No applier means directory is not under quota management
	if applier == nil {
		return nil, nil
	}
	ibytes, err := applier.GetConsumption(path, dirQuotaMap[path])
	if err != nil {
		return nil, err
	}
	return resource.NewQuantity(ibytes, resource.DecimalSI), nil
}


GetInodes

// GetInodes -- retrieve the number of inodes in use under the directory
func GetInodes(path string) (*resource.Quantity, error) {
	// Note that we actually need to hold the lock at least through
	// running the quota command, so it can't get recycled behind our back
	quotaLock.Lock()
	defer quotaLock.Unlock()
	applier := getApplier(path)
	// No applier means directory is not under quota management
	if applier == nil {
		return nil, nil
	}
	inodes, err := applier.GetInodes(path, dirQuotaMap[path])
	if err != nil {
		return nil, err
	}
	return resource.NewQuantity(inodes, resource.DecimalSI), nil
}


ClearQuota

// ClearQuota -- remove the quota assigned to a directory
func ClearQuota(m mount.Interface, path string) error {
	klog.V(3).Infof("ClearQuota %s", path)
	if !enabledQuotasForMonitoring() {
		return fmt.Errorf("clearQuota called, but quotas disabled")
	}
	quotaLock.Lock()
	defer quotaLock.Unlock()
	poduid, ok := dirPodMap[path]
	if !ok {
		// Nothing in the map either means that there was no
		// quota to begin with or that we're clearing a
		// stale directory, so if we find a quota, just remove it.
		// The process of clearing the quota requires that an applier
		// be found, which needs to be cleaned up.
		defer delete(supportsQuotasMap, path)
		defer clearApplier(path)
		return clearQuotaOnDir(m, path)
	}
	_, ok = podQuotaMap[poduid]
	if !ok {
		return fmt.Errorf("clearQuota: No quota available for %s", path)
	}
	projid, err := GetQuotaOnDir(m, path)
	if err != nil {
		// Log-and-continue instead of returning an error for now
		// due to unspecified backwards compatibility concerns (a subject to revise)
		klog.V(3).Infof("Attempt to check quota ID %v on dir %s failed: %v", dirQuotaMap[path], path, err)
	}
	if projid != dirQuotaMap[path] {
		return fmt.Errorf("expected quota ID %v on dir %s does not match actual %v", dirQuotaMap[path], path, projid)
	}
	count, ok := podDirCountMap[poduid]
	if count <= 1 || !ok {
		err = clearQuotaOnDir(m, path)
		// This error should be noted; we still need to clean up
		// and otherwise handle in the same way.
		if err != nil {
			klog.V(3).Infof("Unable to clear quota %v %s: %v", dirQuotaMap[path], path, err)
		}
		delete(quotaSizeMap, podQuotaMap[poduid])
		delete(quotaPodMap, podQuotaMap[poduid])
		delete(podDirCountMap, poduid)
		delete(podQuotaMap, poduid)
		delete(podUidMap, poduid)
	} else {
		err = removeProjectID(path, projid)
		podDirCountMap[poduid]--
		klog.V(4).Infof("Not clearing quota for pod %s; still %v dirs outstanding", poduid, podDirCountMap[poduid])
	}
	delete(dirPodMap, path)
	delete(dirQuotaMap, path)
	delete(supportsQuotasMap, path)
	clearApplier(path)
	if err != nil {
		return fmt.Errorf("unable to clear quota for %s: %v", path, err)
	}
	return nil
}


GetQuotaOnDir

func GetQuotaOnDir(_ mount.Interface, _ string) (common.QuotaID, error) {
	return common.BadQuotaID, errNotImplemented
}


SupportsQuotas

// SupportsQuotas -- dummy implementation
func SupportsQuotas(_ mount.Interface, _ string) (bool, error) {
	return false, errNotImplemented
}


AssignQuota

// AssignQuota -- dummy implementation
func AssignQuota(_ mount.Interface, _ string, _ types.UID, _ *resource.Quantity) error {
	return errNotImplemented
}


GetConsumption

// GetConsumption -- dummy implementation
func GetConsumption(_ string) (*resource.Quantity, error) {
	return nil, errNotImplemented
}


GetInodes

// GetInodes -- dummy implementation
func GetInodes(_ string) (*resource.Quantity, error) {
	return nil, errNotImplemented
}


ClearQuota

// ClearQuota -- dummy implementation
func ClearQuota(_ mount.Interface, _ string) error {
	return errNotImplemented
}


getFileType

// getFileType checks for file/directory/socket and block/character devices.
func getFileType(pathname string) (FileType, error) {
	var pathType FileType
	info, err := os.Stat(pathname)
	if os.IsNotExist(err) {
		return pathType, fmt.Errorf("path %q does not exist", pathname)
	}
	// err in call to os.Stat
	if err != nil {
		return pathType, err
	}

	// checks whether the mode is the target mode.
	isSpecificMode := func(mode, targetMode os.FileMode) bool {
		return mode&targetMode == targetMode
	}

	mode := info.Mode()
	if mode.IsDir() {
		return FileTypeDirectory, nil
	} else if mode.IsRegular() {
		return FileTypeFile, nil
	} else if isSpecificMode(mode, os.ModeSocket) {
		return FileTypeSocket, nil
	} else if isSpecificMode(mode, os.ModeDevice) {
		if isSpecificMode(mode, os.ModeCharDevice) {
			return FileTypeCharDev, nil
		}
		return FileTypeBlockDev, nil
	}

	return pathType, fmt.Errorf("only recognise file, directory, socket, block device and character device")
}


NewHostUtil

// NewHostUtil returns a struct that implements the HostUtils interface on
// linux platforms
func NewHostUtil() *HostUtil {
	return &HostUtil{}
}


DeviceOpened

// DeviceOpened checks if block device in use by calling Open with O_EXCL flag.
// If pathname is not a device, log and return false with nil error.
// If open returns errno EBUSY, return true with nil error.
// If open returns nil, return false with nil error.
// Otherwise, return false with error
func (hu *HostUtil) DeviceOpened(pathname string) (bool, error) {
	return ExclusiveOpenFailsOnDevice(pathname)
}


PathIsDevice

// PathIsDevice uses FileInfo returned from os.Stat to check if path refers
// to a device.
func (hu *HostUtil) PathIsDevice(pathname string) (bool, error) {
	pathType, err := hu.GetFileType(pathname)
	isDevice := pathType == FileTypeCharDev || pathType == FileTypeBlockDev
	return isDevice, err
}


ExclusiveOpenFailsOnDevice

// ExclusiveOpenFailsOnDevice is shared with NsEnterMounter
func ExclusiveOpenFailsOnDevice(pathname string) (bool, error) {
	var isDevice bool
	finfo, err := os.Stat(pathname)
	if os.IsNotExist(err) {
		isDevice = false
	}
	// err in call to os.Stat
	if err != nil {
		return false, fmt.Errorf(
			"PathIsDevice failed for path %q: %v",
			pathname,
			err)
	}
	// path refers to a device
	if finfo.Mode()&os.ModeDevice != 0 {
		isDevice = true
	}

	if !isDevice {
		klog.Errorf("Path %q is not referring to a device.", pathname)
		return false, nil
	}
	fd, errno := unix.Open(pathname, unix.O_RDONLY|unix.O_EXCL|unix.O_CLOEXEC, 0)
	// If the device is in use, open will return an invalid fd.
	// When this happens, it is expected that Close will fail and throw an error.
	defer unix.Close(fd)
	if errno == nil {
		// device not in use
		return false, nil
	} else if errno == unix.EBUSY {
		// device is in use
		return true, nil
	}
	// error during call to Open
	return false, errno
}


GetDeviceNameFromMount

// GetDeviceNameFromMount given a mount point, find the device name from its global mount point
func (hu *HostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
	return getDeviceNameFromMount(mounter, mountPath, pluginMountDir)
}


getDeviceNameFromMount

// getDeviceNameFromMountLinux find the device name from /proc/mounts in which
// the mount path reference should match the given plugin mount directory. In case no mount path reference
// matches, returns the volume name taken from its given mountPath
func getDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
	refs, err := mounter.GetMountRefs(mountPath)
	if err != nil {
		klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err)
		return "", err
	}
	if len(refs) == 0 {
		klog.V(4).Infof("Directory %s is not mounted", mountPath)
		return "", fmt.Errorf("directory %s is not mounted", mountPath)
	}
	for _, ref := range refs {
		if strings.HasPrefix(ref, pluginMountDir) {
			volumeID, err := filepath.Rel(pluginMountDir, ref)
			if err != nil {
				klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err)
				return "", err
			}
			return volumeID, nil
		}
	}

	return path.Base(mountPath), nil
}


MakeRShared

// MakeRShared checks that given path is on a mount with 'rshared' mount
// propagation. If not, it bind-mounts the path as rshared.
func (hu *HostUtil) MakeRShared(path string) error {
	return DoMakeRShared(path, procMountInfoPath)
}


GetFileType

// GetFileType checks for file/directory/socket/block/character devices.
func (hu *HostUtil) GetFileType(pathname string) (FileType, error) {
	return getFileType(pathname)
}


PathExists

// PathExists tests if the given path already exists
// Error is returned on any other error than "file not found".
func (hu *HostUtil) PathExists(pathname string) (bool, error) {
	return utilpath.Exists(utilpath.CheckFollowSymlink, pathname)
}


EvalHostSymlinks

// EvalHostSymlinks returns the path name after evaluating symlinks.
// TODO once the nsenter implementation is removed, this method can be removed
// from the interface and filepath.EvalSymlinks used directly
func (hu *HostUtil) EvalHostSymlinks(pathname string) (string, error) {
	return filepath.EvalSymlinks(pathname)
}


FindMountInfo

// FindMountInfo returns the mount info on the given path.
func (hu *HostUtil) FindMountInfo(path string) (mount.MountInfo, error) {
	return findMountInfo(path, procMountInfoPath)
}


isShared

// isShared returns true, if given path is on a mount point that has shared
// mount propagation.
func isShared(mount string, mountInfoPath string) (bool, error) {
	info, err := findMountInfo(mount, mountInfoPath)
	if err != nil {
		return false, err
	}

	// parse optional parameters
	for _, opt := range info.OptionalFields {
		if strings.HasPrefix(opt, "shared:") {
			return true, nil
		}
	}
	return false, nil
}


findMountInfo

func findMountInfo(path, mountInfoPath string) (mount.MountInfo, error) {
	infos, err := mount.ParseMountInfo(mountInfoPath)
	if err != nil {
		return mount.MountInfo{}, err
	}

	// process /proc/xxx/mountinfo in backward order and find the first mount
	// point that is prefix of 'path' - that's the mount where path resides
	var info *mount.MountInfo
	for i := len(infos) - 1; i >= 0; i-- {
		if mount.PathWithinBase(path, infos[i].MountPoint) {
			info = &infos[i]
			break
		}
	}
	if info == nil {
		return mount.MountInfo{}, fmt.Errorf("cannot find mount point for %q", path)
	}
	return *info, nil
}


DoMakeRShared

// DoMakeRShared is common implementation of MakeRShared on Linux. It checks if
// path is shared and bind-mounts it as rshared if needed. mountCmd and
// mountArgs are expected to contain mount-like command, DoMakeRShared will add
// '--bind <path> <path>' and '--make-rshared <path>' to mountArgs.
func DoMakeRShared(path string, mountInfoFilename string) error {
	shared, err := isShared(path, mountInfoFilename)
	if err != nil {
		return err
	}
	if shared {
		klog.V(4).Infof("Directory %s is already on a shared mount", path)
		return nil
	}

	klog.V(2).Infof("Bind-mounting %q with shared mount propagation", path)
	// mount --bind /var/lib/kubelet /var/lib/kubelet
	if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_BIND, "" /*data*/); err != nil {
		return fmt.Errorf("failed to bind-mount %s: %v", path, err)
	}

	// mount --make-rshared /var/lib/kubelet
	if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_SHARED|syscall.MS_REC, "" /*data*/); err != nil {
		return fmt.Errorf("failed to make %s rshared: %v", path, err)
	}

	return nil
}


GetSELinux

// GetSELinux is common implementation of GetSELinuxSupport on Linux.
func GetSELinux(path string, mountInfoFilename string, selinuxEnabled seLinuxEnabledFunc) (bool, error) {
	// Skip /proc/mounts parsing if SELinux is disabled.
	if !selinuxEnabled() {
		return false, nil
	}

	info, err := findMountInfo(path, mountInfoFilename)
	if err != nil {
		return false, err
	}

	// "seclabel" can be both in mount options and super options.
	for _, opt := range info.SuperOptions {
		if opt == "seclabel" {
			return true, nil
		}
	}
	for _, opt := range info.MountOptions {
		if opt == "seclabel" {
			return true, nil
		}
	}
	return false, nil
}


GetSELinuxSupport

// GetSELinuxSupport returns true if given path is on a mount that supports
// SELinux.
func (hu *HostUtil) GetSELinuxSupport(pathname string) (bool, error) {
	return GetSELinux(pathname, procMountInfoPath, selinux.GetEnabled)
}


GetOwner

// GetOwner returns the integer ID for the user and group of the given path
func (hu *HostUtil) GetOwner(pathname string) (int64, int64, error) {
	realpath, err := filepath.EvalSymlinks(pathname)
	if err != nil {
		return -1, -1, err
	}
	return GetOwnerLinux(realpath)
}


GetMode

// GetMode returns permissions of the path.
func (hu *HostUtil) GetMode(pathname string) (os.FileMode, error) {
	return GetModeLinux(pathname)
}


GetOwnerLinux

// GetOwnerLinux is shared between Linux and NsEnterMounter
// pathname must already be evaluated for symlinks
func GetOwnerLinux(pathname string) (int64, int64, error) {
	info, err := os.Stat(pathname)
	if err != nil {
		return -1, -1, err
	}
	stat := info.Sys().(*syscall.Stat_t)
	return int64(stat.Uid), int64(stat.Gid), nil
}


GetModeLinux

// GetModeLinux is shared between Linux and NsEnterMounter
func GetModeLinux(pathname string) (os.FileMode, error) {
	info, err := os.Stat(pathname)
	if err != nil {
		return 0, err
	}
	return info.Mode(), nil
}


GetSELinuxMountContext

// GetSELinuxMountContext returns value of -o context=XYZ mount option on
// given mount point.
func (hu *HostUtil) GetSELinuxMountContext(pathname string) (string, error) {
	return getSELinuxMountContext(pathname, procMountInfoPath, selinux.GetEnabled)
}


getSELinuxMountContext

// getSELinux is common implementation of GetSELinuxSupport on Linux.
// Using an extra function for unit tests.
func getSELinuxMountContext(path string, mountInfoFilename string, selinuxEnabled seLinuxEnabledFunc) (string, error) {
	// Skip /proc/mounts parsing if SELinux is disabled.
	if !selinuxEnabled() {
		return "", nil
	}

	info, err := findMountInfo(path, mountInfoFilename)
	if err != nil {
		return "", err
	}

	for _, opt := range info.SuperOptions {
		if !strings.HasPrefix(opt, "context=") {
			continue
		}
		// Remove context=
		context := strings.TrimPrefix(opt, "context=")
		// Remove double quotes
		context = strings.Trim(context, "\"")
		return context, nil
	}
	return "", nil
}


NewHostUtil

// NewHostUtil returns a struct that implements the HostUtils interface on
// unsupported platforms
func NewHostUtil() *HostUtil {
	return &HostUtil{}
}


DeviceOpened

// DeviceOpened always returns an error on unsupported platforms
func (hu *HostUtil) DeviceOpened(pathname string) (bool, error) {
	return false, errUnsupported
}


PathIsDevice

// PathIsDevice always returns an error on unsupported platforms
func (hu *HostUtil) PathIsDevice(pathname string) (bool, error) {
	return true, errUnsupported
}


GetDeviceNameFromMount

// GetDeviceNameFromMount always returns an error on unsupported platforms
func (hu *HostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
	return getDeviceNameFromMount(mounter, mountPath, pluginMountDir)
}


MakeRShared

// MakeRShared always returns an error on unsupported platforms
func (hu *HostUtil) MakeRShared(path string) error {
	return errUnsupported
}


GetFileType

// GetFileType always returns an error on unsupported platforms
func (hu *HostUtil) GetFileType(pathname string) (FileType, error) {
	return FileType("fake"), errUnsupported
}


MakeFile

// MakeFile always returns an error on unsupported platforms
func (hu *HostUtil) MakeFile(pathname string) error {
	return errUnsupported
}


MakeDir

// MakeDir always returns an error on unsupported platforms
func (hu *HostUtil) MakeDir(pathname string) error {
	return errUnsupported
}


PathExists

// PathExists always returns an error on unsupported platforms
func (hu *HostUtil) PathExists(pathname string) (bool, error) {
	return true, errUnsupported
}


EvalHostSymlinks

// EvalHostSymlinks always returns an error on unsupported platforms
func (hu *HostUtil) EvalHostSymlinks(pathname string) (string, error) {
	return "", errUnsupported
}


GetOwner

// GetOwner always returns an error on unsupported platforms
func (hu *HostUtil) GetOwner(pathname string) (int64, int64, error) {
	return -1, -1, errUnsupported
}


GetSELinuxSupport

// GetSELinuxSupport always returns an error on unsupported platforms
func (hu *HostUtil) GetSELinuxSupport(pathname string) (bool, error) {
	return false, errUnsupported
}


GetMode

// GetMode always returns an error on unsupported platforms
func (hu *HostUtil) GetMode(pathname string) (os.FileMode, error) {
	return 0, errUnsupported
}


getDeviceNameFromMount

func getDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
	return "", errUnsupported
}


GetSELinuxMountContext

// GetSELinuxMountContext returns value of -o context=XYZ mount option on
// given mount point.
func (hu *HostUtil) GetSELinuxMountContext(pathname string) (string, error) {
	return "", errUnsupported
}


NewHostUtil

// NewHostUtil returns a struct that implements HostUtils on Windows platforms
func NewHostUtil() *HostUtil {
	return &HostUtil{}
}


GetDeviceNameFromMount

// GetDeviceNameFromMount given a mnt point, find the device
func (hu *HostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
	return getDeviceNameFromMount(mounter, mountPath, pluginMountDir)
}


getDeviceNameFromMount

// getDeviceNameFromMount find the device(drive) name in which
// the mount path reference should match the given plugin mount directory. In case no mount path reference
// matches, returns the volume name taken from its given mountPath
func getDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
	refs, err := mounter.GetMountRefs(mountPath)
	if err != nil {
		klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err)
		return "", err
	}
	if len(refs) == 0 {
		return "", fmt.Errorf("directory %s is not mounted", mountPath)
	}
	basemountPath := mount.NormalizeWindowsPath(pluginMountDir)
	for _, ref := range refs {
		if strings.Contains(ref, basemountPath) {
			volumeID, err := filepath.Rel(mount.NormalizeWindowsPath(basemountPath), ref)
			if err != nil {
				klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err)
				return "", err
			}
			return volumeID, nil
		}
	}

	return path.Base(mountPath), nil
}


DeviceOpened

// DeviceOpened determines if the device is in use elsewhere
func (hu *HostUtil) DeviceOpened(pathname string) (bool, error) {
	return false, nil
}


PathIsDevice

// PathIsDevice determines if a path is a device.
func (hu *HostUtil) PathIsDevice(pathname string) (bool, error) {
	return false, nil
}


MakeRShared

// MakeRShared checks that given path is on a mount with 'rshared' mount
// propagation. Empty implementation here.
func (hu *HostUtil) MakeRShared(path string) error {
	return nil
}


GetFileType

// GetFileType checks for sockets/block/character devices
func (hu *(HostUtil)) GetFileType(pathname string) (FileType, error) {
	return getFileType(pathname)
}


PathExists

// PathExists checks whether the path exists
func (hu *HostUtil) PathExists(pathname string) (bool, error) {
	return utilpath.Exists(utilpath.CheckFollowSymlink, pathname)
}


EvalHostSymlinks

// EvalHostSymlinks returns the path name after evaluating symlinks
func (hu *HostUtil) EvalHostSymlinks(pathname string) (string, error) {
	return filepath.EvalSymlinks(pathname)
}


GetOwner

// GetOwner returns the integer ID for the user and group of the given path
// Note that on windows, it always returns 0. We actually don't set Group on
// windows platform, see SetVolumeOwnership implementation.
func (hu *HostUtil) GetOwner(pathname string) (int64, int64, error) {
	return -1, -1, nil
}


GetSELinuxSupport

// GetSELinuxSupport returns a boolean indicating support for SELinux.
// Windows does not support SELinux.
func (hu *HostUtil) GetSELinuxSupport(pathname string) (bool, error) {
	return false, nil
}


GetMode

// GetMode returns permissions of the path.
func (hu *HostUtil) GetMode(pathname string) (os.FileMode, error) {
	info, err := os.Stat(pathname)
	if err != nil {
		return 0, err
	}
	return info.Mode(), nil
}


GetSELinuxMountContext

// GetSELinuxMountContext returns value of -o context=XYZ mount option on
// given mount point.
func (hu *HostUtil) GetSELinuxMountContext(pathname string) (string, error) {
	return "", nil
}


NewIOHandler

// NewIOHandler Create a new IoHandler implementation
func NewIOHandler() IoUtil {
	return &osIOHandler{}
}


ReadFile

func (handler *osIOHandler) ReadFile(filename string) ([]byte, error) {
	return ioutil.ReadFile(filename)
}


ReadDir

func (handler *osIOHandler) ReadDir(dirname string) ([]os.FileInfo, error) {
	return ioutil.ReadDir(dirname)
}


Lstat

func (handler *osIOHandler) Lstat(name string) (os.FileInfo, error) {
	return os.Lstat(name)
}


EvalSymlinks

func (handler *osIOHandler) EvalSymlinks(path string) (string, error) {
	return filepath.EvalSymlinks(path)
}


registerMetrics

func registerMetrics() {
	// legacyregistry is the internal k8s wrapper around the prometheus
	// global registry, used specifically for metric stability enforcement
	legacyregistry.MustRegister(StorageOperationMetric)
	legacyregistry.MustRegister(storageOperationEndToEndLatencyMetric)
	legacyregistry.MustRegister(csiOperationsLatencyMetric)
}


OperationCompleteHook

// OperationCompleteHook returns a hook to call when an operation is completed
func OperationCompleteHook(plugin, operationName string) func(types.CompleteFuncParam) {
	requestTime := time.Now()
	opComplete := func(c types.CompleteFuncParam) {
		timeTaken := time.Since(requestTime).Seconds()
		// Create metric with operation name and plugin name
		status := statusSuccess
		if *c.Err != nil {
			// TODO: Establish well-known error codes to be able to distinguish
			// user configuration errors from system errors.
			status = statusFailUnknown
		}
		migrated := false
		if c.Migrated != nil {
			migrated = *c.Migrated
		}
		StorageOperationMetric.WithLabelValues(plugin, operationName, status, strconv.FormatBool(migrated)).Observe(timeTaken)
	}
	return opComplete
}


FSGroupCompleteHook

// FSGroupCompleteHook returns a hook to call when volume recursive permission is changed
func FSGroupCompleteHook(plugin volume.VolumePlugin, spec *volume.Spec) func(types.CompleteFuncParam) {
	return OperationCompleteHook(GetFullQualifiedPluginNameForVolume(plugin.GetPluginName(), spec), "volume_apply_access_control")
}


GetFullQualifiedPluginNameForVolume

// GetFullQualifiedPluginNameForVolume returns full qualified plugin name for
// given volume. For CSI plugin, it appends plugin driver name at the end of
// plugin name, e.g. kubernetes.io/csi:csi-hostpath. It helps to distinguish
// between metrics emitted for CSI volumes which may be handled by different
// CSI plugin drivers.
func GetFullQualifiedPluginNameForVolume(pluginName string, spec *volume.Spec) string {
	if spec != nil {
		if spec.Volume != nil && spec.Volume.CSI != nil {
			return fmt.Sprintf("%s:%s", pluginName, spec.Volume.CSI.Driver)
		}
		if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.CSI != nil {
			return fmt.Sprintf("%s:%s", pluginName, spec.PersistentVolume.Spec.CSI.Driver)
		}
	}
	return pluginName
}


RecordOperationLatencyMetric

// RecordOperationLatencyMetric records the end to end latency for certain operation
// into metric volume_operation_total_seconds
func RecordOperationLatencyMetric(plugin, operationName string, secondsTaken float64) {
	storageOperationEndToEndLatencyMetric.WithLabelValues(plugin, operationName).Observe(secondsTaken)
}


RecordCSIOperationLatencyMetrics

// RecordCSIOperationLatencyMetrics records the CSI operation latency and grpc status
// into metric csi_kubelet_operations_seconds
func RecordCSIOperationLatencyMetrics(driverName string,
	operationName string,
	operationErr error,
	operationDuration time.Duration,
	migrated string) {
	csiOperationsLatencyMetric.WithLabelValues(driverName, operationName, getErrorCode(operationErr), migrated).Observe(operationDuration.Seconds())
}


getErrorCode

func getErrorCode(err error) string {
	if err == nil {
		return codes.OK.String()
	}

	st, ok := status.FromError(err)
	if !ok {
		// This is not gRPC error. The operation must have failed before gRPC
		// method was called, otherwise we would get gRPC error.
		return "unknown-non-grpc"
	}

	return st.Code().String()
}


getNestedMountpoints

// getNestedMountpoints returns a list of mountpoint directories that should be created
// for the volume indicated by name.
// note: the returned list is relative to baseDir
func getNestedMountpoints(name, baseDir string, pod v1.Pod) ([]string, error) {
	var retval []string
	checkContainer := func(container *v1.Container) error {
		var allMountPoints []string // all mount points in this container
		var myMountPoints []string  // mount points that match name
		for _, vol := range container.VolumeMounts {
			cleaned := filepath.Clean(vol.MountPath)
			allMountPoints = append(allMountPoints, cleaned)
			if vol.Name == name {
				myMountPoints = append(myMountPoints, cleaned)
			}
		}
		sort.Strings(allMountPoints)
		parentPrefix := ".." + string(os.PathSeparator)
		// Examine each place where this volume is mounted
		for _, myMountPoint := range myMountPoints {
			if strings.HasPrefix(myMountPoint, parentPrefix) {
				// Don't let a container trick us into creating directories outside of its rootfs
				return fmt.Errorf("invalid container mount point %v", myMountPoint)
			}
			myMPSlash := myMountPoint + string(os.PathSeparator)
			// The previously found nested mountpoints.
			// NOTE: We can't simply rely on sort.Strings to have all the mountpoints sorted and
			// grouped. For example, the following strings are sorted in this exact order:
			// /dir/nested, /dir/nested-vol, /dir/nested.vol, /dir/nested/double, /dir/nested2
			// The issue is a bit worse for Windows paths, since the \'s value is higher than /'s:
			// \dir\nested, \dir\nested-vol, \dir\nested.vol, \dir\nested2, \dir\nested\double
			// Because of this, we should use a list of previously mounted mountpoints, rather than only one.
			prevNestedMPs := []string{}
			// examine each mount point to see if it's nested beneath this volume
			// (but skip any that are double-nested beneath this volume)
			// For example, if this volume is mounted as /dir and other volumes are mounted
			//              as /dir/nested and /dir/nested/other, only create /dir/nested.
			for _, mp := range allMountPoints {
				if !strings.HasPrefix(mp, myMPSlash) {
					continue // skip -- not nested beneath myMountPoint
				}

				isNested := false
				for _, prevNestedMP := range prevNestedMPs {
					if strings.HasPrefix(mp, prevNestedMP) {
						isNested = true
						break
					}
				}
				if isNested {
					continue // skip -- double nested beneath myMountPoint
				}
				// since this mount point is nested, remember it so that we can check that following ones aren't nested beneath this one
				prevNestedMPs = append(prevNestedMPs, mp+string(os.PathSeparator))
				retval = append(retval, mp[len(myMPSlash):])
			}
		}
		return nil
	}

	var retErr error
	podutil.VisitContainers(&pod.Spec, podutil.AllFeatureEnabledContainers(), func(c *v1.Container, containerType podutil.ContainerType) bool {
		retErr = checkContainer(c)
		return retErr == nil
	})
	if retErr != nil {
		return nil, retErr
	}

	return retval, nil
}


MakeNestedMountpoints

// MakeNestedMountpoints creates mount points in baseDir for volumes mounted beneath name
func MakeNestedMountpoints(name, baseDir string, pod v1.Pod) error {
	dirs, err := getNestedMountpoints(name, baseDir, pod)
	if err != nil {
		return err
	}
	for _, dir := range dirs {
		err := os.MkdirAll(filepath.Join(baseDir, dir), 0755)
		if err != nil {
			return fmt.Errorf("unable to create nested volume mountpoints: %v", err)
		}
	}
	return nil
}


NewNestedPendingOperations

// NewNestedPendingOperations returns a new instance of NestedPendingOperations.
func NewNestedPendingOperations(exponentialBackOffOnError bool) NestedPendingOperations {
	g := &nestedPendingOperations{
		operations:                []operation{},
		exponentialBackOffOnError: exponentialBackOffOnError,
	}
	g.cond = sync.NewCond(&g.lock)
	return g
}


Run

func (grm *nestedPendingOperations) Run(
	volumeName v1.UniqueVolumeName,
	podName volumetypes.UniquePodName,
	nodeName types.NodeName,
	generatedOperations volumetypes.GeneratedOperations) error {
	grm.lock.Lock()
	defer grm.lock.Unlock()

	opKey := operationKey{volumeName, podName, nodeName}

	opExists, previousOpIndex := grm.isOperationExists(opKey)
	if opExists {
		previousOp := grm.operations[previousOpIndex]
		// Operation already exists
		if previousOp.operationPending {
			// Operation is pending
			return NewAlreadyExistsError(opKey)
		}

		backOffErr := previousOp.expBackoff.SafeToRetry(fmt.Sprintf("%+v", opKey))
		if backOffErr != nil {
			if previousOp.operationName == generatedOperations.OperationName {
				return backOffErr
			}
			// previous operation and new operation are different. reset op. name and exp. backoff
			grm.operations[previousOpIndex].operationName = generatedOperations.OperationName
			grm.operations[previousOpIndex].expBackoff = exponentialbackoff.ExponentialBackoff{}
		}

		// Update existing operation to mark as pending.
		grm.operations[previousOpIndex].operationPending = true
		grm.operations[previousOpIndex].key = opKey
	} else {
		// Create a new operation
		grm.operations = append(grm.operations,
			operation{
				key:              opKey,
				operationPending: true,
				operationName:    generatedOperations.OperationName,
				expBackoff:       exponentialbackoff.ExponentialBackoff{},
			})
	}

	go func() (eventErr, detailedErr error) {
		// Handle unhandled panics (very unlikely)
		defer k8sRuntime.HandleCrash()
		// Handle completion of and error, if any, from operationFunc()
		defer grm.operationComplete(opKey, &detailedErr)
		return generatedOperations.Run()
	}()

	return nil
}


IsOperationSafeToRetry

func (grm *nestedPendingOperations) IsOperationSafeToRetry(
	volumeName v1.UniqueVolumeName,
	podName volumetypes.UniquePodName,
	nodeName types.NodeName,
	operationName string) bool {

	grm.lock.RLock()
	defer grm.lock.RUnlock()

	opKey := operationKey{volumeName, podName, nodeName}
	exist, previousOpIndex := grm.isOperationExists(opKey)
	if !exist {
		return true
	}
	previousOp := grm.operations[previousOpIndex]
	if previousOp.operationPending {
		return false
	}
	backOffErr := previousOp.expBackoff.SafeToRetry(fmt.Sprintf("%+v", opKey))
	if backOffErr != nil {
		if previousOp.operationName == operationName {
			return false
		}
	}

	return true
}


IsOperationPending

func (grm *nestedPendingOperations) IsOperationPending(
	volumeName v1.UniqueVolumeName,
	podName volumetypes.UniquePodName,
	nodeName types.NodeName) bool {

	grm.lock.RLock()
	defer grm.lock.RUnlock()

	opKey := operationKey{volumeName, podName, nodeName}
	exist, previousOpIndex := grm.isOperationExists(opKey)
	if exist && grm.operations[previousOpIndex].operationPending {
		return true
	}
	return false
}


isOperationExists

// This is an internal function and caller should acquire and release the lock
func (grm *nestedPendingOperations) isOperationExists(key operationKey) (bool, int) {

	// If volumeName is empty, operation can be executed concurrently
	if key.volumeName == EmptyUniqueVolumeName {
		return false, -1
	}

	opIndex := -1
	for previousOpIndex, previousOp := range grm.operations {
		volumeNameMatch := previousOp.key.volumeName == key.volumeName

		podNameMatch := previousOp.key.podName == EmptyUniquePodName ||
			key.podName == EmptyUniquePodName ||
			previousOp.key.podName == key.podName

		podNameExactMatch := previousOp.key.podName == key.podName

		nodeNameMatch := previousOp.key.nodeName == EmptyNodeName ||
			key.nodeName == EmptyNodeName ||
			previousOp.key.nodeName == key.nodeName

		nodeNameExactMatch := previousOp.key.nodeName == key.nodeName

		if volumeNameMatch && podNameMatch && nodeNameMatch {
			// nonExactMatch pending first
			if previousOp.operationPending {
				return true, previousOpIndex
			}
			// nonExactMatch with no pending, set opIndex to the first nonExactMatch
			// exactMatch can override opIndex to expected
			if opIndex == -1 || (podNameExactMatch && nodeNameExactMatch) {
				opIndex = previousOpIndex
			}
		}
	}
	return opIndex != -1, opIndex

}


getOperation

func (grm *nestedPendingOperations) getOperation(key operationKey) (uint, error) {
	// Assumes lock has been acquired by caller.

	for i, op := range grm.operations {
		if op.key.volumeName == key.volumeName &&
			op.key.podName == key.podName &&
			op.key.nodeName == key.nodeName {
			return uint(i), nil
		}
	}

	return 0, fmt.Errorf("operation %+v not found", key)
}


deleteOperation

func (grm *nestedPendingOperations) deleteOperation(key operationKey) {
	// Assumes lock has been acquired by caller.

	opIndex := -1
	for i, op := range grm.operations {
		if op.key.volumeName == key.volumeName &&
			op.key.podName == key.podName &&
			op.key.nodeName == key.nodeName {
			opIndex = i
			break
		}
	}

	if opIndex < 0 {
		return
	}

	// Delete index without preserving order
	grm.operations[opIndex] = grm.operations[len(grm.operations)-1]
	grm.operations = grm.operations[:len(grm.operations)-1]
}


operationComplete

func (grm *nestedPendingOperations) operationComplete(key operationKey, err *error) {
	// Defer operations are executed in Last-In is First-Out order. In this case
	// the lock is acquired first when operationCompletes begins, and is
	// released when the method finishes, after the lock is released cond is
	// signaled to wake waiting goroutine.
	defer grm.cond.Signal()
	grm.lock.Lock()
	defer grm.lock.Unlock()

	if *err == nil || !grm.exponentialBackOffOnError {
		// Operation completed without error, or exponentialBackOffOnError disabled
		grm.deleteOperation(key)
		if *err != nil {
			// Log error
			klog.Errorf("operation %+v failed with: %v", key, *err)
		}
		return
	}

	// Operation completed with error and exponentialBackOffOnError Enabled
	existingOpIndex, getOpErr := grm.getOperation(key)
	if getOpErr != nil {
		// Failed to find existing operation
		klog.Errorf("Operation %+v completed. error: %v. exponentialBackOffOnError is enabled, but failed to get operation to update.",
			key,
			*err)
		return
	}

	grm.operations[existingOpIndex].expBackoff.Update(err)
	grm.operations[existingOpIndex].operationPending = false

	// Log error
	klog.Errorf("%v", grm.operations[existingOpIndex].expBackoff.
		GenerateNoRetriesPermittedMsg(fmt.Sprintf("%+v", key)))
}


Wait

func (grm *nestedPendingOperations) Wait() {
	grm.lock.Lock()
	defer grm.lock.Unlock()

	for len(grm.operations) > 0 {
		grm.cond.Wait()
	}
}


NewAlreadyExistsError

// NewAlreadyExistsError returns a new instance of AlreadyExists error.
func NewAlreadyExistsError(key operationKey) error {
	return alreadyExistsError{key}
}


IsAlreadyExists

// IsAlreadyExists returns true if an error returned from
// NestedPendingOperations indicates a new operation can not be started because
// an operation with the same operation name is already executing.
func IsAlreadyExists(err error) bool {
	switch err.(type) {
	case alreadyExistsError:
		return true
	default:
		return false
	}
}


Error

func (err alreadyExistsError) Error() string {
	return fmt.Sprintf(
		"Failed to create operation with name %+v. An operation with that name is already executing.",
		err.operationKey)
}


newNodeExpander

func newNodeExpander(resizeOp nodeResizeOperationOpts, client clientset.Interface, recorder record.EventRecorder) *NodeExpander {
	return &NodeExpander{
		kubeClient:              client,
		nodeResizeOperationOpts: resizeOp,
		recorder:                recorder,
	}
}


runPreCheck

// runPreCheck performs some sanity checks before expansion can be performed on the PVC.
func (ne *NodeExpander) runPreCheck() bool {
	ne.pvcStatusCap = ne.pvc.Status.Capacity[v1.ResourceStorage]
	ne.pvCap = ne.pv.Spec.Capacity[v1.ResourceStorage]

	ne.resizeStatus = ne.pvc.Status.ResizeStatus

	// PVC is already expanded but we are still trying to expand the volume because
	// last recorded size in ASOW is older. This can happen for RWX volume types.
	if ne.pvcStatusCap.Cmp(ne.pluginResizeOpts.NewSize) >= 0 && (ne.resizeStatus == nil || *ne.resizeStatus == v1.PersistentVolumeClaimNoExpansionInProgress) {
		ne.pvcAlreadyUpdated = true
	}

	// if resizestatus is nil or NodeExpansionInProgress or NodeExpansionPending then we
	// should allow volume expansion on the node to proceed. We are making an exception for
	// resizeStatus being nil because it will support use cases where
	// resizeStatus may not be set (old control-plane expansion controller etc).
	if ne.resizeStatus == nil ||
		ne.pvcAlreadyUpdated ||
		*ne.resizeStatus == v1.PersistentVolumeClaimNodeExpansionPending ||
		*ne.resizeStatus == v1.PersistentVolumeClaimNodeExpansionInProgress {
		return true
	}

	return false
}


expandOnPlugin

func (ne *NodeExpander) expandOnPlugin() (bool, error, testResponseData) {
	allowExpansion := ne.runPreCheck()
	if !allowExpansion {
		return false, nil, testResponseData{false, true}
	}

	var err error
	nodeName := ne.vmt.Pod.Spec.NodeName

	if !ne.pvcAlreadyUpdated {
		ne.pvc, err = util.MarkNodeExpansionInProgress(ne.pvc, ne.kubeClient)

		if err != nil {
			msg := ne.vmt.GenerateErrorDetailed("MountVolume.NodeExpandVolume failed to mark node expansion in progress: %v", err)
			klog.Errorf(msg.Error())
			return false, err, testResponseData{}
		}
	}
	_, resizeErr := ne.volumePlugin.NodeExpand(ne.pluginResizeOpts)
	if resizeErr != nil {
		if volumetypes.IsOperationFinishedError(resizeErr) {
			var markFailedError error
			ne.pvc, markFailedError = util.MarkNodeExpansionFailed(ne.pvc, ne.kubeClient)
			if markFailedError != nil {
				klog.Errorf(ne.vmt.GenerateErrorDetailed("MountMount.NodeExpandVolume failed to mark node expansion as failed: %v", err).Error())
			}
		}

		// if driver returned FailedPrecondition error that means
		// volume expansion should not be retried on this node but
		// expansion operation should not block mounting
		if volumetypes.IsFailedPreconditionError(resizeErr) {
			ne.actualStateOfWorld.MarkForInUseExpansionError(ne.vmt.VolumeName)
			klog.Errorf(ne.vmt.GenerateErrorDetailed("MountVolume.NodeExapndVolume failed with %v", resizeErr).Error())
			return false, nil, testResponseData{assumeResizeFinished: true, resizeCalledOnPlugin: true}
		}
		return false, resizeErr, testResponseData{assumeResizeFinished: true, resizeCalledOnPlugin: true}
	}
	simpleMsg, detailedMsg := ne.vmt.GenerateMsg("MountVolume.NodeExpandVolume succeeded", nodeName)
	ne.recorder.Eventf(ne.vmt.Pod, v1.EventTypeNormal, kevents.FileSystemResizeSuccess, simpleMsg)
	ne.recorder.Eventf(ne.pvc, v1.EventTypeNormal, kevents.FileSystemResizeSuccess, simpleMsg)
	klog.InfoS(detailedMsg, "pod", klog.KObj(ne.vmt.Pod))

	// no need to update PVC object if we already updated it
	if ne.pvcAlreadyUpdated {
		return true, nil, testResponseData{true, true}
	}

	// File system resize succeeded, now update the PVC's Capacity to match the PV's
	ne.pvc, err = util.MarkFSResizeFinished(ne.pvc, ne.pluginResizeOpts.NewSize, ne.kubeClient)
	if err != nil {
		return true, fmt.Errorf("mountVolume.NodeExpandVolume update pvc status failed: %v", err), testResponseData{true, true}
	}
	return true, nil, testResponseData{true, true}
}


NewOperationExecutor

// NewOperationExecutor returns a new instance of OperationExecutor.
func NewOperationExecutor(
	operationGenerator OperationGenerator) OperationExecutor {

	return &operationExecutor{
		pendingOperations: nestedpendingoperations.NewNestedPendingOperations(
			true /* exponentialBackOffOnError */),
		operationGenerator: operationGenerator,
	}
}


errSuffix

// Generates an error string with the format ": <err>" if err exists
func errSuffix(err error) string {
	errStr := ""
	if err != nil {
		errStr = fmt.Sprintf(": %v", err)
	}
	return errStr
}


generateVolumeMsgDetailed

// Generate a detailed error msg for logs
func generateVolumeMsgDetailed(prefixMsg, suffixMsg, volumeName, details string) (detailedMsg string) {
	return fmt.Sprintf("%v for volume %q %v %v", prefixMsg, volumeName, details, suffixMsg)
}


generateVolumeMsg

// Generate a simplified error msg for events and a detailed error msg for logs
func generateVolumeMsg(prefixMsg, suffixMsg, volumeName, details string) (simpleMsg, detailedMsg string) {
	simpleMsg = fmt.Sprintf("%v for volume %q %v", prefixMsg, volumeName, suffixMsg)
	return simpleMsg, generateVolumeMsgDetailed(prefixMsg, suffixMsg, volumeName, details)
}


GenerateMsgDetailed

// GenerateMsgDetailed returns detailed msgs for volumes to attach
func (volume *VolumeToAttach) GenerateMsgDetailed(prefixMsg, suffixMsg string) (detailedMsg string) {
	detailedStr := fmt.Sprintf("(UniqueName: %q) from node %q", volume.VolumeName, volume.NodeName)
	volumeSpecName := "nil"
	if volume.VolumeSpec != nil {
		volumeSpecName = volume.VolumeSpec.Name()
	}
	return generateVolumeMsgDetailed(prefixMsg, suffixMsg, volumeSpecName, detailedStr)
}


GenerateMsg

// GenerateMsg returns simple and detailed msgs for volumes to attach
func (volume *VolumeToAttach) GenerateMsg(prefixMsg, suffixMsg string) (simpleMsg, detailedMsg string) {
	detailedStr := fmt.Sprintf("(UniqueName: %q) from node %q", volume.VolumeName, volume.NodeName)
	volumeSpecName := "nil"
	if volume.VolumeSpec != nil {
		volumeSpecName = volume.VolumeSpec.Name()
	}
	return generateVolumeMsg(prefixMsg, suffixMsg, volumeSpecName, detailedStr)
}


GenerateErrorDetailed

// GenerateErrorDetailed returns detailed errors for volumes to attach
func (volume *VolumeToAttach) GenerateErrorDetailed(prefixMsg string, err error) (detailedErr error) {
	return fmt.Errorf(volume.GenerateMsgDetailed(prefixMsg, errSuffix(err)))
}


GenerateError

// GenerateError returns simple and detailed errors for volumes to attach
func (volume *VolumeToAttach) GenerateError(prefixMsg string, err error) (simpleErr, detailedErr error) {
	simpleMsg, detailedMsg := volume.GenerateMsg(prefixMsg, errSuffix(err))
	return fmt.Errorf(simpleMsg), fmt.Errorf(detailedMsg)
}


String

// String combines key fields of the volume for logging in text format.
func (volume *VolumeToAttach) String() string {
	volumeSpecName := "nil"
	if volume.VolumeSpec != nil {
		volumeSpecName = volume.VolumeSpec.Name()
	}
	return fmt.Sprintf("%s (UniqueName: %s) from node %s", volumeSpecName, volume.VolumeName, volume.NodeName)
}


MarshalLog

// MarshalLog combines key fields of the volume for logging in a structured format.
func (volume *VolumeToAttach) MarshalLog() interface{} {
	volumeSpecName := "nil"
	if volume.VolumeSpec != nil {
		volumeSpecName = volume.VolumeSpec.Name()
	}
	return struct {
		VolumeName, UniqueName, NodeName string
	}{
		VolumeName: volumeSpecName,
		UniqueName: string(volume.VolumeName),
		NodeName:   string(volume.NodeName),
	}
}


Error

func (err *MountPreConditionFailed) Error() string {
	return err.msg
}


NewMountPreConditionFailedError

func NewMountPreConditionFailedError(msg string) *MountPreConditionFailed {
	return &MountPreConditionFailed{msg: msg}
}


IsMountFailedPreconditionError

func IsMountFailedPreconditionError(err error) bool {
	var failedPreconditionError *MountPreConditionFailed
	return errors.As(err, &failedPreconditionError)
}


GenerateMsgDetailed

// GenerateMsgDetailed returns detailed msgs for volumes to mount
func (volume *VolumeToMount) GenerateMsgDetailed(prefixMsg, suffixMsg string) (detailedMsg string) {
	detailedStr := fmt.Sprintf("(UniqueName: %q) pod %q (UID: %q)", volume.VolumeName, volume.Pod.Name, volume.Pod.UID)
	volumeSpecName := "nil"
	if volume.VolumeSpec != nil {
		volumeSpecName = volume.VolumeSpec.Name()
	}
	return generateVolumeMsgDetailed(prefixMsg, suffixMsg, volumeSpecName, detailedStr)
}


GenerateMsg

// GenerateMsg returns simple and detailed msgs for volumes to mount
func (volume *VolumeToMount) GenerateMsg(prefixMsg, suffixMsg string) (simpleMsg, detailedMsg string) {
	detailedStr := fmt.Sprintf("(UniqueName: %q) pod %q (UID: %q)", volume.VolumeName, volume.Pod.Name, volume.Pod.UID)
	volumeSpecName := "nil"
	if volume.VolumeSpec != nil {
		volumeSpecName = volume.VolumeSpec.Name()
	}
	return generateVolumeMsg(prefixMsg, suffixMsg, volumeSpecName, detailedStr)
}


GenerateErrorDetailed

// GenerateErrorDetailed returns detailed errors for volumes to mount
func (volume *VolumeToMount) GenerateErrorDetailed(prefixMsg string, err error) (detailedErr error) {
	return fmt.Errorf(volume.GenerateMsgDetailed(prefixMsg, errSuffix(err)))
}


GenerateError

// GenerateError returns simple and detailed errors for volumes to mount
func (volume *VolumeToMount) GenerateError(prefixMsg string, err error) (simpleErr, detailedErr error) {
	simpleMsg, detailedMsg := volume.GenerateMsg(prefixMsg, errSuffix(err))
	return fmt.Errorf(simpleMsg), fmt.Errorf(detailedMsg)
}


GenerateMsgDetailed

// GenerateMsgDetailed returns detailed msgs for attached volumes
func (volume *AttachedVolume) GenerateMsgDetailed(prefixMsg, suffixMsg string) (detailedMsg string) {
	detailedStr := fmt.Sprintf("(UniqueName: %q) on node %q", volume.VolumeName, volume.NodeName)
	volumeSpecName := "nil"
	if volume.VolumeSpec != nil {
		volumeSpecName = volume.VolumeSpec.Name()
	}
	return generateVolumeMsgDetailed(prefixMsg, suffixMsg, volumeSpecName, detailedStr)
}


GenerateMsg

// GenerateMsg returns simple and detailed msgs for attached volumes
func (volume *AttachedVolume) GenerateMsg(prefixMsg, suffixMsg string) (simpleMsg, detailedMsg string) {
	detailedStr := fmt.Sprintf("(UniqueName: %q) on node %q", volume.VolumeName, volume.NodeName)
	volumeSpecName := "nil"
	if volume.VolumeSpec != nil {
		volumeSpecName = volume.VolumeSpec.Name()
	}
	return generateVolumeMsg(prefixMsg, suffixMsg, volumeSpecName, detailedStr)
}


GenerateErrorDetailed

// GenerateErrorDetailed returns detailed errors for attached volumes
func (volume *AttachedVolume) GenerateErrorDetailed(prefixMsg string, err error) (detailedErr error) {
	return fmt.Errorf(volume.GenerateMsgDetailed(prefixMsg, errSuffix(err)))
}


GenerateError

// GenerateError returns simple and detailed errors for attached volumes
func (volume *AttachedVolume) GenerateError(prefixMsg string, err error) (simpleErr, detailedErr error) {
	simpleMsg, detailedMsg := volume.GenerateMsg(prefixMsg, errSuffix(err))
	return fmt.Errorf(simpleMsg), fmt.Errorf(detailedMsg)
}


String

// String combines key fields of the volume for logging in text format.
func (volume *AttachedVolume) String() string {
	volumeSpecName := "nil"
	if volume.VolumeSpec != nil {
		volumeSpecName = volume.VolumeSpec.Name()
	}
	return fmt.Sprintf("%s (UniqueName: %s) from node %s", volumeSpecName, volume.VolumeName, volume.NodeName)
}


MarshalLog

// MarshalLog combines key fields of the volume for logging in a structured format.
func (volume *AttachedVolume) MarshalLog() interface{} {
	volumeSpecName := "nil"
	if volume.VolumeSpec != nil {
		volumeSpecName = volume.VolumeSpec.Name()
	}
	return struct {
		VolumeName, UniqueName, NodeName string
	}{
		VolumeName: volumeSpecName,
		UniqueName: string(volume.VolumeName),
		NodeName:   string(volume.NodeName),
	}
}


GenerateMsgDetailed

// GenerateMsgDetailed returns detailed msgs for mounted volumes
func (volume *MountedVolume) GenerateMsgDetailed(prefixMsg, suffixMsg string) (detailedMsg string) {
	detailedStr := fmt.Sprintf("(UniqueName: %q) pod %q (UID: %q)", volume.VolumeName, volume.PodName, volume.PodUID)
	return generateVolumeMsgDetailed(prefixMsg, suffixMsg, volume.OuterVolumeSpecName, detailedStr)
}


GenerateMsg

// GenerateMsg returns simple and detailed msgs for mounted volumes
func (volume *MountedVolume) GenerateMsg(prefixMsg, suffixMsg string) (simpleMsg, detailedMsg string) {
	detailedStr := fmt.Sprintf("(UniqueName: %q) pod %q (UID: %q)", volume.VolumeName, volume.PodName, volume.PodUID)
	return generateVolumeMsg(prefixMsg, suffixMsg, volume.OuterVolumeSpecName, detailedStr)
}


GenerateErrorDetailed

// GenerateErrorDetailed returns simple and detailed errors for mounted volumes
func (volume *MountedVolume) GenerateErrorDetailed(prefixMsg string, err error) (detailedErr error) {
	return fmt.Errorf(volume.GenerateMsgDetailed(prefixMsg, errSuffix(err)))
}


GenerateError

// GenerateError returns simple and detailed errors for mounted volumes
func (volume *MountedVolume) GenerateError(prefixMsg string, err error) (simpleErr, detailedErr error) {
	simpleMsg, detailedMsg := volume.GenerateMsg(prefixMsg, errSuffix(err))
	return fmt.Errorf(simpleMsg), fmt.Errorf(detailedMsg)
}


IsOperationPending

func (oe *operationExecutor) IsOperationPending(
	volumeName v1.UniqueVolumeName,
	podName volumetypes.UniquePodName,
	nodeName types.NodeName) bool {
	return oe.pendingOperations.IsOperationPending(volumeName, podName, nodeName)
}


IsOperationSafeToRetry

func (oe *operationExecutor) IsOperationSafeToRetry(
	volumeName v1.UniqueVolumeName,
	podName volumetypes.UniquePodName,
	nodeName types.NodeName,
	operationName string) bool {
	return oe.pendingOperations.IsOperationSafeToRetry(volumeName, podName, nodeName, operationName)
}


AttachVolume

func (oe *operationExecutor) AttachVolume(
	logger klog.Logger,
	volumeToAttach VolumeToAttach,
	actualStateOfWorld ActualStateOfWorldAttacherUpdater) error {
	generatedOperations :=
		oe.operationGenerator.GenerateAttachVolumeFunc(logger, volumeToAttach, actualStateOfWorld)

	if util.IsMultiAttachAllowed(volumeToAttach.VolumeSpec) {
		return oe.pendingOperations.Run(
			volumeToAttach.VolumeName, "" /* podName */, volumeToAttach.NodeName, generatedOperations)
	}

	return oe.pendingOperations.Run(
		volumeToAttach.VolumeName, "" /* podName */, "" /* nodeName */, generatedOperations)
}


DetachVolume

func (oe *operationExecutor) DetachVolume(
	logger klog.Logger,
	volumeToDetach AttachedVolume,
	verifySafeToDetach bool,
	actualStateOfWorld ActualStateOfWorldAttacherUpdater) error {
	generatedOperations, err :=
		oe.operationGenerator.GenerateDetachVolumeFunc(logger, volumeToDetach, verifySafeToDetach, actualStateOfWorld)
	if err != nil {
		return err
	}

	if util.IsMultiAttachAllowed(volumeToDetach.VolumeSpec) {
		return oe.pendingOperations.Run(
			volumeToDetach.VolumeName, "" /* podName */, volumeToDetach.NodeName, generatedOperations)
	}
	return oe.pendingOperations.Run(
		volumeToDetach.VolumeName, "" /* podName */, "" /* nodeName */, generatedOperations)

}


VerifyVolumesAreAttached

func (oe *operationExecutor) VerifyVolumesAreAttached(
	attachedVolumes map[types.NodeName][]AttachedVolume,
	actualStateOfWorld ActualStateOfWorldAttacherUpdater) {

	// A map of plugin names and nodes on which they exist with volumes they manage
	bulkVerifyPluginsByNode := make(map[string]map[types.NodeName][]*volume.Spec)
	volumeSpecMapByPlugin := make(map[string]map[*volume.Spec]v1.UniqueVolumeName)

	for node, nodeAttachedVolumes := range attachedVolumes {
		needIndividualVerifyVolumes := []AttachedVolume{}
		for _, volumeAttached := range nodeAttachedVolumes {
			if volumeAttached.VolumeSpec == nil {
				klog.Errorf("VerifyVolumesAreAttached: nil spec for volume %s", volumeAttached.VolumeName)
				continue
			}

			volumePlugin, err :=
				oe.operationGenerator.GetVolumePluginMgr().FindPluginBySpec(volumeAttached.VolumeSpec)
			if err != nil {
				klog.Errorf(
					"VolumesAreAttached.FindPluginBySpec failed for volume %q (spec.Name: %q) on node %q with error: %v",
					volumeAttached.VolumeName,
					volumeAttached.VolumeSpec.Name(),
					volumeAttached.NodeName,
					err)
				continue
			}
			if volumePlugin == nil {
				// should never happen since FindPluginBySpec always returns error if volumePlugin = nil
				klog.Errorf(
					"Failed to find volume plugin for volume %q (spec.Name: %q) on node %q",
					volumeAttached.VolumeName,
					volumeAttached.VolumeSpec.Name(),
					volumeAttached.NodeName)
				continue
			}

			pluginName := volumePlugin.GetPluginName()

			if volumePlugin.SupportsBulkVolumeVerification() {
				pluginNodes, pluginNodesExist := bulkVerifyPluginsByNode[pluginName]

				if !pluginNodesExist {
					pluginNodes = make(map[types.NodeName][]*volume.Spec)
				}

				volumeSpecList, nodeExists := pluginNodes[node]
				if !nodeExists {
					volumeSpecList = []*volume.Spec{}
				}
				volumeSpecList = append(volumeSpecList, volumeAttached.VolumeSpec)
				pluginNodes[node] = volumeSpecList

				bulkVerifyPluginsByNode[pluginName] = pluginNodes
				volumeSpecMap, mapExists := volumeSpecMapByPlugin[pluginName]

				if !mapExists {
					volumeSpecMap = make(map[*volume.Spec]v1.UniqueVolumeName)
				}
				volumeSpecMap[volumeAttached.VolumeSpec] = volumeAttached.VolumeName
				volumeSpecMapByPlugin[pluginName] = volumeSpecMap
				continue
			}
			// If node doesn't support Bulk volume polling it is best to poll individually
			needIndividualVerifyVolumes = append(needIndividualVerifyVolumes, volumeAttached)
		}
		nodeError := oe.VerifyVolumesAreAttachedPerNode(needIndividualVerifyVolumes, node, actualStateOfWorld)
		if nodeError != nil {
			klog.Errorf("VerifyVolumesAreAttached failed for volumes %v, node %q with error %v", needIndividualVerifyVolumes, node, nodeError)
		}
	}

	for pluginName, pluginNodeVolumes := range bulkVerifyPluginsByNode {
		generatedOperations, err := oe.operationGenerator.GenerateBulkVolumeVerifyFunc(
			pluginNodeVolumes,
			pluginName,
			volumeSpecMapByPlugin[pluginName],
			actualStateOfWorld)
		if err != nil {
			klog.Errorf("BulkVerifyVolumes.GenerateBulkVolumeVerifyFunc error bulk verifying volumes for plugin %q with  %v", pluginName, err)
		}

		// Ugly hack to ensure - we don't do parallel bulk polling of same volume plugin
		uniquePluginName := v1.UniqueVolumeName(pluginName)
		err = oe.pendingOperations.Run(uniquePluginName, "" /* Pod Name */, "" /* nodeName */, generatedOperations)
		if err != nil {
			klog.Errorf("BulkVerifyVolumes.Run Error bulk volume verification for plugin %q  with %v", pluginName, err)
		}
	}
}


VerifyVolumesAreAttachedPerNode

func (oe *operationExecutor) VerifyVolumesAreAttachedPerNode(
	attachedVolumes []AttachedVolume,
	nodeName types.NodeName,
	actualStateOfWorld ActualStateOfWorldAttacherUpdater) error {
	generatedOperations, err :=
		oe.operationGenerator.GenerateVolumesAreAttachedFunc(attachedVolumes, nodeName, actualStateOfWorld)
	if err != nil {
		return err
	}

	// Give an empty UniqueVolumeName so that this operation could be executed concurrently.
	return oe.pendingOperations.Run("" /* volumeName */, "" /* podName */, "" /* nodeName */, generatedOperations)
}


MountVolume

func (oe *operationExecutor) MountVolume(
	waitForAttachTimeout time.Duration,
	volumeToMount VolumeToMount,
	actualStateOfWorld ActualStateOfWorldMounterUpdater,
	isRemount bool) error {
	fsVolume, err := util.CheckVolumeModeFilesystem(volumeToMount.VolumeSpec)
	if err != nil {
		return err
	}
	var generatedOperations volumetypes.GeneratedOperations
	if fsVolume {
		// Filesystem volume case
		// Mount/remount a volume when a volume is attached
		generatedOperations = oe.operationGenerator.GenerateMountVolumeFunc(
			waitForAttachTimeout, volumeToMount, actualStateOfWorld, isRemount)

	} else {
		// Block volume case
		// Creates a map to device if a volume is attached
		generatedOperations, err = oe.operationGenerator.GenerateMapVolumeFunc(
			waitForAttachTimeout, volumeToMount, actualStateOfWorld)
	}
	if err != nil {
		return err
	}
	// Avoid executing mount/map from multiple pods referencing the
	// same volume in parallel
	podName := nestedpendingoperations.EmptyUniquePodName

	// TODO: remove this -- not necessary
	if !volumeToMount.PluginIsAttachable && !volumeToMount.PluginIsDeviceMountable {
		// volume plugins which are Non-attachable and Non-deviceMountable can execute mount for multiple pods
		// referencing the same volume in parallel
		podName = util.GetUniquePodName(volumeToMount.Pod)
	}

	// TODO mount_device
	return oe.pendingOperations.Run(
		volumeToMount.VolumeName, podName, "" /* nodeName */, generatedOperations)
}


UnmountVolume

func (oe *operationExecutor) UnmountVolume(
	volumeToUnmount MountedVolume,
	actualStateOfWorld ActualStateOfWorldMounterUpdater,
	podsDir string) error {
	fsVolume, err := util.CheckVolumeModeFilesystem(volumeToUnmount.VolumeSpec)
	if err != nil {
		return err
	}
	var generatedOperations volumetypes.GeneratedOperations
	if fsVolume {
		// Filesystem volume case
		// Unmount a volume if a volume is mounted
		generatedOperations, err = oe.operationGenerator.GenerateUnmountVolumeFunc(
			volumeToUnmount, actualStateOfWorld, podsDir)
	} else {
		// Block volume case
		// Unmap a volume if a volume is mapped
		generatedOperations, err = oe.operationGenerator.GenerateUnmapVolumeFunc(
			volumeToUnmount, actualStateOfWorld)
	}
	if err != nil {
		return err
	}
	// All volume plugins can execute unmount/unmap for multiple pods referencing the
	// same volume in parallel
	podName := volumetypes.UniquePodName(volumeToUnmount.PodUID)

	return oe.pendingOperations.Run(
		volumeToUnmount.VolumeName, podName, "" /* nodeName */, generatedOperations)
}


UnmountDevice

func (oe *operationExecutor) UnmountDevice(
	deviceToDetach AttachedVolume,
	actualStateOfWorld ActualStateOfWorldMounterUpdater,
	hostutil hostutil.HostUtils) error {
	fsVolume, err := util.CheckVolumeModeFilesystem(deviceToDetach.VolumeSpec)
	if err != nil {
		return err
	}
	var generatedOperations volumetypes.GeneratedOperations
	if fsVolume {
		// Filesystem volume case
		// Unmount and detach a device if a volume isn't referenced
		generatedOperations, err = oe.operationGenerator.GenerateUnmountDeviceFunc(
			deviceToDetach, actualStateOfWorld, hostutil)
	} else {
		// Block volume case
		// Detach a device and remove loopback if a volume isn't referenced
		generatedOperations, err = oe.operationGenerator.GenerateUnmapDeviceFunc(
			deviceToDetach, actualStateOfWorld, hostutil)
	}
	if err != nil {
		return err
	}
	// Avoid executing unmount/unmap device from multiple pods referencing
	// the same volume in parallel
	podName := nestedpendingoperations.EmptyUniquePodName

	return oe.pendingOperations.Run(
		deviceToDetach.VolumeName, podName, "" /* nodeName */, generatedOperations)
}


ExpandInUseVolume

func (oe *operationExecutor) ExpandInUseVolume(volumeToMount VolumeToMount, actualStateOfWorld ActualStateOfWorldMounterUpdater, currentSize resource.Quantity) error {
	generatedOperations, err := oe.operationGenerator.GenerateExpandInUseVolumeFunc(volumeToMount, actualStateOfWorld, currentSize)
	if err != nil {
		return err
	}
	return oe.pendingOperations.Run(volumeToMount.VolumeName, "", "" /* nodeName */, generatedOperations)
}


VerifyControllerAttachedVolume

func (oe *operationExecutor) VerifyControllerAttachedVolume(
	logger klog.Logger,
	volumeToMount VolumeToMount,
	nodeName types.NodeName,
	actualStateOfWorld ActualStateOfWorldAttacherUpdater) error {
	generatedOperations, err :=
		oe.operationGenerator.GenerateVerifyControllerAttachedVolumeFunc(logger, volumeToMount, nodeName, actualStateOfWorld)
	if err != nil {
		return err
	}

	return oe.pendingOperations.Run(
		volumeToMount.VolumeName, "" /* podName */, "" /* nodeName */, generatedOperations)
}


ReconstructVolumeOperation

// ReconstructVolumeOperation return a func to create volumeSpec from mount path
func (oe *operationExecutor) ReconstructVolumeOperation(
	volumeMode v1.PersistentVolumeMode,
	plugin volume.VolumePlugin,
	mapperPlugin volume.BlockVolumePlugin,
	uid types.UID,
	podName volumetypes.UniquePodName,
	volumeSpecName string,
	volumePath string,
	pluginName string) (volume.ReconstructedVolume, error) {

	// Filesystem Volume case
	if volumeMode == v1.PersistentVolumeFilesystem {
		// Create volumeSpec from mount path
		klog.V(5).Infof("Starting operationExecutor.ReconstructVolume for file volume on pod %q", podName)
		reconstructed, err := plugin.ConstructVolumeSpec(volumeSpecName, volumePath)
		if err != nil {
			return volume.ReconstructedVolume{}, err
		}
		return reconstructed, nil
	}

	// Block Volume case
	// Create volumeSpec from mount path
	klog.V(5).Infof("Starting operationExecutor.ReconstructVolume for block volume on pod %q", podName)

	// volumePath contains volumeName on the path. In the case of block volume, {volumeName} is symbolic link
	// corresponding to raw block device.
	// ex. volumePath: pods/{podUid}}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName}
	volumeSpec, err := mapperPlugin.ConstructBlockVolumeSpec(uid, volumeSpecName, volumePath)
	if err != nil {
		return volume.ReconstructedVolume{}, err
	}
	return volume.ReconstructedVolume{
		Spec: volumeSpec,
	}, nil
}


CheckVolumeExistenceOperation

// CheckVolumeExistenceOperation checks mount path directory if volume still exists
func (oe *operationExecutor) CheckVolumeExistenceOperation(
	volumeSpec *volume.Spec,
	mountPath, volumeName string,
	mounter mount.Interface,
	uniqueVolumeName v1.UniqueVolumeName,
	podName volumetypes.UniquePodName,
	podUID types.UID,
	attachable volume.AttachableVolumePlugin) (bool, error) {
	fsVolume, err := util.CheckVolumeModeFilesystem(volumeSpec)
	if err != nil {
		return false, err
	}

	// Filesystem Volume case
	// For attachable volume case, check mount path directory if volume is still existing and mounted.
	// Return true if volume is mounted.
	if fsVolume {
		if attachable != nil {
			var isNotMount bool
			var mountCheckErr error
			if mounter == nil {
				return false, fmt.Errorf("mounter was not set for a filesystem volume")
			}
			if isNotMount, mountCheckErr = mount.IsNotMountPoint(mounter, mountPath); mountCheckErr != nil {
				return false, fmt.Errorf("could not check whether the volume %q (spec.Name: %q) pod %q (UID: %q) is mounted with: %v",
					uniqueVolumeName,
					volumeName,
					podName,
					podUID,
					mountCheckErr)
			}
			return !isNotMount, nil
		}
		// For non-attachable volume case, skip check and return true without mount point check
		// since plugins may not have volume mount point.
		return true, nil
	}

	// Block Volume case
	// Check mount path directory if volume still exists, then return true if volume
	// is there. Either plugin is attachable or non-attachable, the plugin should
	// have symbolic link associated to raw block device under pod device map
	// if volume exists.
	blkutil := volumepathhandler.NewBlockVolumePathHandler()
	var islinkExist bool
	var checkErr error
	if islinkExist, checkErr = blkutil.IsSymlinkExist(mountPath); checkErr != nil {
		return false, fmt.Errorf("could not check whether the block volume %q (spec.Name: %q) pod %q (UID: %q) is mapped to: %v",
			uniqueVolumeName,
			volumeName,
			podName,
			podUID,
			checkErr)
	}
	return islinkExist, nil
}


NewOperationGenerator

// NewOperationGenerator is returns instance of operationGenerator
func NewOperationGenerator(kubeClient clientset.Interface,
	volumePluginMgr *volume.VolumePluginMgr,
	recorder record.EventRecorder,
	blkUtil volumepathhandler.BlockVolumePathHandler) OperationGenerator {

	return &operationGenerator{
		kubeClient:      kubeClient,
		volumePluginMgr: volumePluginMgr,
		recorder:        recorder,
		blkUtil:         blkUtil,
		translator:      csitrans.New(),
	}
}


GenerateVolumesAreAttachedFunc

func (og *operationGenerator) GenerateVolumesAreAttachedFunc(
	attachedVolumes []AttachedVolume,
	nodeName types.NodeName,
	actualStateOfWorld ActualStateOfWorldAttacherUpdater) (volumetypes.GeneratedOperations, error) {
	// volumesPerPlugin maps from a volume plugin to a list of volume specs which belong
	// to this type of plugin
	volumesPerPlugin := make(map[string][]*volume.Spec)
	// volumeSpecMap maps from a volume spec to its unique volumeName which will be used
	// when calling MarkVolumeAsDetached
	volumeSpecMap := make(map[*volume.Spec]v1.UniqueVolumeName)

	// Iterate each volume spec and put them into a map index by the pluginName
	for _, volumeAttached := range attachedVolumes {
		if volumeAttached.VolumeSpec == nil {
			klog.Errorf("VerifyVolumesAreAttached.GenerateVolumesAreAttachedFunc: nil spec for volume %s", volumeAttached.VolumeName)
			continue
		}
		volumePlugin, err :=
			og.volumePluginMgr.FindPluginBySpec(volumeAttached.VolumeSpec)
		if err != nil || volumePlugin == nil {
			klog.Errorf(volumeAttached.GenerateErrorDetailed("VolumesAreAttached.FindPluginBySpec failed", err).Error())
			continue
		}
		volumeSpecList, pluginExists := volumesPerPlugin[volumePlugin.GetPluginName()]
		if !pluginExists {
			volumeSpecList = []*volume.Spec{}
		}
		volumeSpecList = append(volumeSpecList, volumeAttached.VolumeSpec)
		volumesPerPlugin[volumePlugin.GetPluginName()] = volumeSpecList
		// Migration: VolumeSpecMap contains original VolumeName for use in ActualStateOfWorld
		volumeSpecMap[volumeAttached.VolumeSpec] = volumeAttached.VolumeName
	}

	volumesAreAttachedFunc := func() volumetypes.OperationContext {

		// For each volume plugin, pass the list of volume specs to VolumesAreAttached to check
		// whether the volumes are still attached.
		for pluginName, volumesSpecs := range volumesPerPlugin {
			attachableVolumePlugin, err :=
				og.volumePluginMgr.FindAttachablePluginByName(pluginName)
			if err != nil || attachableVolumePlugin == nil {
				klog.Errorf(
					"VolumeAreAttached.FindAttachablePluginBySpec failed for plugin %q with: %v",
					pluginName,
					err)
				continue
			}

			volumeAttacher, newAttacherErr := attachableVolumePlugin.NewAttacher()
			if newAttacherErr != nil {
				klog.Errorf(
					"VolumesAreAttached.NewAttacher failed for getting plugin %q with: %v",
					pluginName,
					newAttacherErr)
				continue
			}

			attached, areAttachedErr := volumeAttacher.VolumesAreAttached(volumesSpecs, nodeName)
			if areAttachedErr != nil {
				klog.Errorf(
					"VolumesAreAttached failed for checking on node %q with: %v",
					nodeName,
					areAttachedErr)
				continue
			}

			for spec, check := range attached {
				if !check {
					actualStateOfWorld.MarkVolumeAsDetached(volumeSpecMap[spec], nodeName)
					klog.V(1).Infof("VerifyVolumesAreAttached determined volume %q (spec.Name: %q) is no longer attached to node %q, therefore it was marked as detached.",
						volumeSpecMap[spec], spec.Name(), nodeName)
				}
			}
		}

		// It is hard to differentiate migrated status for all volumes for verify_volumes_are_attached_per_node
		return volumetypes.NewOperationContext(nil, nil, false)
	}

	return volumetypes.GeneratedOperations{
		OperationName:     "verify_volumes_are_attached_per_node",
		OperationFunc:     volumesAreAttachedFunc,
		CompleteFunc:      util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume("<n/a>", nil), "verify_volumes_are_attached_per_node"),
		EventRecorderFunc: nil, // nil because we do not want to generate event on error
	}, nil
}


GenerateBulkVolumeVerifyFunc

func (og *operationGenerator) GenerateBulkVolumeVerifyFunc(
	pluginNodeVolumes map[types.NodeName][]*volume.Spec,
	pluginName string,
	volumeSpecMap map[*volume.Spec]v1.UniqueVolumeName,
	actualStateOfWorld ActualStateOfWorldAttacherUpdater) (volumetypes.GeneratedOperations, error) {

	// Migration: All inputs already should be translated by caller for this
	// function except volumeSpecMap which contains original volume names for
	// use with actualStateOfWorld

	bulkVolumeVerifyFunc := func() volumetypes.OperationContext {
		attachableVolumePlugin, err :=
			og.volumePluginMgr.FindAttachablePluginByName(pluginName)
		if err != nil || attachableVolumePlugin == nil {
			klog.Errorf(
				"BulkVerifyVolume.FindAttachablePluginBySpec failed for plugin %q with: %v",
				pluginName,
				err)
			return volumetypes.NewOperationContext(nil, nil, false)
		}

		volumeAttacher, newAttacherErr := attachableVolumePlugin.NewAttacher()

		if newAttacherErr != nil {
			klog.Errorf(
				"BulkVerifyVolume.NewAttacher failed for getting plugin %q with: %v",
				attachableVolumePlugin,
				newAttacherErr)
			return volumetypes.NewOperationContext(nil, nil, false)
		}
		bulkVolumeVerifier, ok := volumeAttacher.(volume.BulkVolumeVerifier)

		if !ok {
			klog.Errorf("BulkVerifyVolume failed to type assert attacher %q", bulkVolumeVerifier)
			return volumetypes.NewOperationContext(nil, nil, false)
		}

		attached, bulkAttachErr := bulkVolumeVerifier.BulkVerifyVolumes(pluginNodeVolumes)
		if bulkAttachErr != nil {
			klog.Errorf("BulkVerifyVolume.BulkVerifyVolumes Error checking volumes are attached with %v", bulkAttachErr)
			return volumetypes.NewOperationContext(nil, nil, false)
		}

		for nodeName, volumeSpecs := range pluginNodeVolumes {
			for _, volumeSpec := range volumeSpecs {
				nodeVolumeSpecs, nodeChecked := attached[nodeName]

				if !nodeChecked {
					klog.V(2).Infof("VerifyVolumesAreAttached.BulkVerifyVolumes failed for node %q and leaving volume %q as attached",
						nodeName,
						volumeSpec.Name())
					continue
				}

				check := nodeVolumeSpecs[volumeSpec]

				if !check {
					klog.V(2).Infof("VerifyVolumesAreAttached.BulkVerifyVolumes failed for node %q and volume %q",
						nodeName,
						volumeSpec.Name())
					actualStateOfWorld.MarkVolumeAsDetached(volumeSpecMap[volumeSpec], nodeName)
				}
			}
		}

		// It is hard to differentiate migrated status for all volumes for verify_volumes_are_attached
		return volumetypes.NewOperationContext(nil, nil, false)
	}

	return volumetypes.GeneratedOperations{
		OperationName:     "verify_volumes_are_attached",
		OperationFunc:     bulkVolumeVerifyFunc,
		CompleteFunc:      util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume(pluginName, nil), "verify_volumes_are_attached"),
		EventRecorderFunc: nil, // nil because we do not want to generate event on error
	}, nil

}


GenerateAttachVolumeFunc

func (og *operationGenerator) GenerateAttachVolumeFunc(
	logger klog.Logger,
	volumeToAttach VolumeToAttach,
	actualStateOfWorld ActualStateOfWorldAttacherUpdater) volumetypes.GeneratedOperations {

	attachVolumeFunc := func() volumetypes.OperationContext {
		attachableVolumePlugin, err :=
			og.volumePluginMgr.FindAttachablePluginBySpec(volumeToAttach.VolumeSpec)

		migrated := getMigratedStatusBySpec(volumeToAttach.VolumeSpec)

		if err != nil || attachableVolumePlugin == nil {
			eventErr, detailedErr := volumeToAttach.GenerateError("AttachVolume.FindAttachablePluginBySpec failed", err)
			return volumetypes.NewOperationContext(eventErr, detailedErr, migrated)
		}

		volumeAttacher, newAttacherErr := attachableVolumePlugin.NewAttacher()
		if newAttacherErr != nil {
			eventErr, detailedErr := volumeToAttach.GenerateError("AttachVolume.NewAttacher failed", newAttacherErr)
			return volumetypes.NewOperationContext(eventErr, detailedErr, migrated)
		}

		// Execute attach
		devicePath, attachErr := volumeAttacher.Attach(
			volumeToAttach.VolumeSpec, volumeToAttach.NodeName)

		if attachErr != nil {
			uncertainNode := volumeToAttach.NodeName
			if derr, ok := attachErr.(*volerr.DanglingAttachError); ok {
				uncertainNode = derr.CurrentNode
			}
			addErr := actualStateOfWorld.MarkVolumeAsUncertain(
				logger,
				volumeToAttach.VolumeName,
				volumeToAttach.VolumeSpec,
				uncertainNode)
			if addErr != nil {
				klog.Errorf("AttachVolume.MarkVolumeAsUncertain fail to add the volume %q to actual state with %s", volumeToAttach.VolumeName, addErr)
			}

			// On failure, return error. Caller will log and retry.
			eventErr, detailedErr := volumeToAttach.GenerateError("AttachVolume.Attach failed", attachErr)
			return volumetypes.NewOperationContext(eventErr, detailedErr, migrated)
		}

		// Successful attach event is useful for user debugging
		simpleMsg, _ := volumeToAttach.GenerateMsg("AttachVolume.Attach succeeded", "")
		for _, pod := range volumeToAttach.ScheduledPods {
			og.recorder.Eventf(pod, v1.EventTypeNormal, kevents.SuccessfulAttachVolume, simpleMsg)
		}
		klog.Infof(volumeToAttach.GenerateMsgDetailed("AttachVolume.Attach succeeded", ""))

		// Update actual state of world
		addVolumeNodeErr := actualStateOfWorld.MarkVolumeAsAttached(
			logger, v1.UniqueVolumeName(""), volumeToAttach.VolumeSpec, volumeToAttach.NodeName, devicePath)
		if addVolumeNodeErr != nil {
			// On failure, return error. Caller will log and retry.
			eventErr, detailedErr := volumeToAttach.GenerateError("AttachVolume.MarkVolumeAsAttached failed", addVolumeNodeErr)
			return volumetypes.NewOperationContext(eventErr, detailedErr, migrated)
		}

		return volumetypes.NewOperationContext(nil, nil, migrated)
	}

	eventRecorderFunc := func(err *error) {
		if *err != nil {
			for _, pod := range volumeToAttach.ScheduledPods {
				og.recorder.Eventf(pod, v1.EventTypeWarning, kevents.FailedAttachVolume, (*err).Error())
			}
		}
	}

	attachableVolumePluginName := unknownAttachableVolumePlugin

	// Get attacher plugin
	attachableVolumePlugin, err :=
		og.volumePluginMgr.FindAttachablePluginBySpec(volumeToAttach.VolumeSpec)
	// It's ok to ignore the error, returning error is not expected from this function.
	// If an error case occurred during the function generation, this error case(skipped one) will also trigger an error
	// while the generated function is executed. And those errors will be handled during the execution of the generated
	// function with a back off policy.
	if err == nil && attachableVolumePlugin != nil {
		attachableVolumePluginName = attachableVolumePlugin.GetPluginName()
	}

	return volumetypes.GeneratedOperations{
		OperationName:     "volume_attach",
		OperationFunc:     attachVolumeFunc,
		EventRecorderFunc: eventRecorderFunc,
		CompleteFunc:      util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume(attachableVolumePluginName, volumeToAttach.VolumeSpec), "volume_attach"),
	}
}


GetVolumePluginMgr

func (og *operationGenerator) GetVolumePluginMgr() *volume.VolumePluginMgr {
	return og.volumePluginMgr
}


GetCSITranslator

func (og *operationGenerator) GetCSITranslator() InTreeToCSITranslator {
	return og.translator
}


GenerateDetachVolumeFunc

func (og *operationGenerator) GenerateDetachVolumeFunc(
	logger klog.Logger,
	volumeToDetach AttachedVolume,
	verifySafeToDetach bool,
	actualStateOfWorld ActualStateOfWorldAttacherUpdater) (volumetypes.GeneratedOperations, error) {
	var volumeName string
	var attachableVolumePlugin volume.AttachableVolumePlugin
	var pluginName string
	var err error

	if volumeToDetach.VolumeSpec != nil {
		attachableVolumePlugin, err = findDetachablePluginBySpec(volumeToDetach.VolumeSpec, og.volumePluginMgr)
		if err != nil || attachableVolumePlugin == nil {
			return volumetypes.GeneratedOperations{}, volumeToDetach.GenerateErrorDetailed("DetachVolume.findDetachablePluginBySpec failed", err)
		}

		volumeName, err =
			attachableVolumePlugin.GetVolumeName(volumeToDetach.VolumeSpec)
		if err != nil {
			return volumetypes.GeneratedOperations{}, volumeToDetach.GenerateErrorDetailed("DetachVolume.GetVolumeName failed", err)
		}
	} else {
		// Get attacher plugin and the volumeName by splitting the volume unique name in case
		// there's no VolumeSpec: this happens only on attach/detach controller crash recovery
		// when a pod has been deleted during the controller downtime
		pluginName, volumeName, err = util.SplitUniqueName(volumeToDetach.VolumeName)
		if err != nil {
			return volumetypes.GeneratedOperations{}, volumeToDetach.GenerateErrorDetailed("DetachVolume.SplitUniqueName failed", err)
		}

		attachableVolumePlugin, err = og.volumePluginMgr.FindAttachablePluginByName(pluginName)
		if err != nil || attachableVolumePlugin == nil {
			return volumetypes.GeneratedOperations{}, volumeToDetach.GenerateErrorDetailed("DetachVolume.FindAttachablePluginByName failed", err)
		}

	}

	if pluginName == "" {
		pluginName = attachableVolumePlugin.GetPluginName()
	}

	volumeDetacher, err := attachableVolumePlugin.NewDetacher()
	if err != nil {
		return volumetypes.GeneratedOperations{}, volumeToDetach.GenerateErrorDetailed("DetachVolume.NewDetacher failed", err)
	}

	detachVolumeFunc := func() volumetypes.OperationContext {
		var err error
		if verifySafeToDetach {
			err = og.verifyVolumeIsSafeToDetach(volumeToDetach)
		}
		if err == nil {
			err = volumeDetacher.Detach(volumeName, volumeToDetach.NodeName)
		}

		migrated := getMigratedStatusBySpec(volumeToDetach.VolumeSpec)

		if err != nil {
			// On failure, add volume back to ReportAsAttached list
			actualStateOfWorld.AddVolumeToReportAsAttached(
				logger, volumeToDetach.VolumeName, volumeToDetach.NodeName)
			eventErr, detailedErr := volumeToDetach.GenerateError("DetachVolume.Detach failed", err)
			return volumetypes.NewOperationContext(eventErr, detailedErr, migrated)
		}

		klog.Infof(volumeToDetach.GenerateMsgDetailed("DetachVolume.Detach succeeded", ""))

		// Update actual state of world
		actualStateOfWorld.MarkVolumeAsDetached(
			volumeToDetach.VolumeName, volumeToDetach.NodeName)

		return volumetypes.NewOperationContext(nil, nil, migrated)
	}

	return volumetypes.GeneratedOperations{
		OperationName:     DetachOperationName,
		OperationFunc:     detachVolumeFunc,
		CompleteFunc:      util.OperationCompleteHook(util.GetFullQualifiedPluginNameForVolume(pluginName, volumeToDetach.VolumeSpec), DetachOperationName),
		EventRecorderFunc: nil, // nil because we do not want to generate event on error
	}, nil
}