func setMigrationAnnotation(migratedPlugins map[string](func() bool), nodeInfo *storagev1.CSINode) (modified bool) {
	if migratedPlugins == nil {
		return false
	}

	nodeInfoAnnotations := nodeInfo.GetAnnotations()
	if nodeInfoAnnotations == nil {
		nodeInfoAnnotations = map[string]string{}
	}

	var oldAnnotationSet sets.String
	mpa := nodeInfoAnnotations[v1.MigratedPluginsAnnotationKey]
	tok := strings.Split(mpa, ",")
	if len(mpa) == 0 {
		oldAnnotationSet = sets.NewString()
	} else {
		oldAnnotationSet = sets.NewString(tok...)
	}

	newAnnotationSet := sets.NewString()
	for pluginName, migratedFunc := range migratedPlugins {
		if migratedFunc() {
			newAnnotationSet.Insert(pluginName)
		}
	}

	if oldAnnotationSet.Equal(newAnnotationSet) {
		return false
	}

	nas := strings.Join(newAnnotationSet.List(), ",")
	if len(nas) != 0 {
		nodeInfoAnnotations[v1.MigratedPluginsAnnotationKey] = nas
	} else {
		delete(nodeInfoAnnotations, v1.MigratedPluginsAnnotationKey)
	}

	nodeInfo.Annotations = nodeInfoAnnotations
	return true
}


// Returns true if and only if new maxAttachLimit doesn't require CSINode update
func keepAllocatableCount(driverInfoSpec storagev1.CSINodeDriver, maxAttachLimit int64) bool {
	if maxAttachLimit == 0 {
		return driverInfoSpec.Allocatable == nil || driverInfoSpec.Allocatable.Count == nil
	}

	return driverInfoSpec.Allocatable != nil && driverInfoSpec.Allocatable.Count != nil && int64(*driverInfoSpec.Allocatable.Count) == maxAttachLimit
}


func (nim *nodeInfoManager) installDriverToCSINode(
	nodeInfo *storagev1.CSINode,
	driverName string,
	driverNodeID string,
	maxAttachLimit int64,
	topology map[string]string) error {

	csiKubeClient := nim.volumeHost.GetKubeClient()
	if csiKubeClient == nil {
		return fmt.Errorf("error getting CSI client")
	}

	topologyKeys := make(sets.String)
	for k := range topology {
		topologyKeys.Insert(k)
	}

	specModified := true
	// Clone driver list, omitting the driver that matches the given driverName
	newDriverSpecs := []storagev1.CSINodeDriver{}
	for _, driverInfoSpec := range nodeInfo.Spec.Drivers {
		if driverInfoSpec.Name == driverName {
			if driverInfoSpec.NodeID == driverNodeID &&
				sets.NewString(driverInfoSpec.TopologyKeys...).Equal(topologyKeys) &&
				keepAllocatableCount(driverInfoSpec, maxAttachLimit) {
				specModified = false
			}
		} else {
			// Omit driverInfoSpec matching given driverName
			newDriverSpecs = append(newDriverSpecs, driverInfoSpec)
		}
	}

	annotationModified := setMigrationAnnotation(nim.migratedPlugins, nodeInfo)

	if !specModified && !annotationModified {
		return nil
	}

	// Append new driver
	driverSpec := storagev1.CSINodeDriver{
		Name:         driverName,
		NodeID:       driverNodeID,
		TopologyKeys: topologyKeys.List(),
	}

	if maxAttachLimit > 0 {
		if maxAttachLimit > math.MaxInt32 {
			klog.Warningf("Exceeded max supported attach limit value, truncating it to %d", math.MaxInt32)
			maxAttachLimit = math.MaxInt32
		}
		m := int32(maxAttachLimit)
		driverSpec.Allocatable = &storagev1.VolumeNodeResources{Count: &m}
	} else if maxAttachLimit != 0 {
		klog.Errorf("Invalid attach limit value %d cannot be added to CSINode object for %q", maxAttachLimit, driverName)
	}

	newDriverSpecs = append(newDriverSpecs, driverSpec)
	nodeInfo.Spec.Drivers = newDriverSpecs

	_, err := csiKubeClient.StorageV1().CSINodes().Update(context.TODO(), nodeInfo, metav1.UpdateOptions{})
	return err
}


func (nim *nodeInfoManager) uninstallDriverFromCSINode(
	csiDriverName string) error {

	csiKubeClient := nim.volumeHost.GetKubeClient()
	if csiKubeClient == nil {
		return fmt.Errorf("error getting CSI client")
	}

	var updateErrs []error
	err := wait.ExponentialBackoff(updateBackoff, func() (bool, error) {
		if err := nim.tryUninstallDriverFromCSINode(csiKubeClient, csiDriverName); err != nil {
			updateErrs = append(updateErrs, err)
			return false, nil
		}
		return true, nil
	})
	if err != nil {
		return fmt.Errorf("error updating CSINode: %v; caused by: %v", err, utilerrors.NewAggregate(updateErrs))
	}
	return nil
}


func (nim *nodeInfoManager) tryUninstallDriverFromCSINode(
	csiKubeClient clientset.Interface,
	csiDriverName string) error {

	nodeInfoClient := csiKubeClient.StorageV1().CSINodes()
	nodeInfo, err := nodeInfoClient.Get(context.TODO(), string(nim.nodeName), metav1.GetOptions{})
	if err != nil && errors.IsNotFound(err) {
		return nil
	} else if err != nil {
		return err
	}

	hasModified := false
	// Uninstall CSINodeDriver with name csiDriverName
	drivers := nodeInfo.Spec.Drivers[:0]
	for _, driver := range nodeInfo.Spec.Drivers {
		if driver.Name != csiDriverName {
			drivers = append(drivers, driver)
		} else {
			// Found a driver with name csiDriverName
			// Set hasModified to true because it will be removed
			hasModified = true
		}
	}

	if !hasModified {
		// No changes, don't update
		return nil
	}
	nodeInfo.Spec.Drivers = drivers

	_, err = nodeInfoClient.Update(context.TODO(), nodeInfo, metav1.UpdateOptions{})

	return err // do not wrap error

}


func removeMaxAttachLimit(driverName string) nodeUpdateFunc {
	return func(node *v1.Node) (*v1.Node, bool, error) {
		limitKey := v1.ResourceName(util.GetCSIAttachLimitKey(driverName))

		capacityExists := false
		if node.Status.Capacity != nil {
			_, capacityExists = node.Status.Capacity[limitKey]
		}

		allocatableExists := false
		if node.Status.Allocatable != nil {
			_, allocatableExists = node.Status.Allocatable[limitKey]
		}

		if !capacityExists && !allocatableExists {
			return node, false, nil
		}

		delete(node.Status.Capacity, limitKey)
		if len(node.Status.Capacity) == 0 {
			node.Status.Capacity = nil
		}

		delete(node.Status.Allocatable, limitKey)
		if len(node.Status.Allocatable) == 0 {
			node.Status.Allocatable = nil
		}

		return node, true, nil
	}
}


// NewPluginManager returns a new PluginManager instance
func NewPluginManager(m PluginNameMapper, featureGate featuregate.FeatureGate) PluginManager {
	return PluginManager{
		PluginNameMapper: m,
		featureGate:      featureGate,
	}
}


// IsMigrationCompleteForPlugin indicates whether CSI migration has been completed
// for a particular storage plugin. A complete migration will need to:
// 1. Enable CSIMigrationXX for the plugin
// 2. Unregister the in-tree plugin by setting the InTreePluginXXUnregister feature gate
func (pm PluginManager) IsMigrationCompleteForPlugin(pluginName string) bool {
	// CSIMigration feature and plugin specific InTreePluginUnregister feature flags should
	// be enabled for plugin specific migration completion to be take effect
	if !pm.IsMigrationEnabledForPlugin(pluginName) {
		return false
	}

	switch pluginName {
	case csilibplugins.AWSEBSInTreePluginName:
		return pm.featureGate.Enabled(features.InTreePluginAWSUnregister)
	case csilibplugins.GCEPDInTreePluginName:
		return pm.featureGate.Enabled(features.InTreePluginGCEUnregister)
	case csilibplugins.AzureFileInTreePluginName:
		return pm.featureGate.Enabled(features.InTreePluginAzureFileUnregister)
	case csilibplugins.AzureDiskInTreePluginName:
		return pm.featureGate.Enabled(features.InTreePluginAzureDiskUnregister)
	case csilibplugins.CinderInTreePluginName:
		return pm.featureGate.Enabled(features.InTreePluginOpenStackUnregister)
	case csilibplugins.VSphereInTreePluginName:
		return pm.featureGate.Enabled(features.InTreePluginvSphereUnregister)
	case csilibplugins.PortworxVolumePluginName:
		return pm.featureGate.Enabled(features.InTreePluginPortworxUnregister)
	case csilibplugins.RBDVolumePluginName:
		return pm.featureGate.Enabled(features.InTreePluginRBDUnregister)
	default:
		return false
	}
}


// IsMigrationEnabledForPlugin indicates whether CSI migration has been enabled
// for a particular storage plugin
func (pm PluginManager) IsMigrationEnabledForPlugin(pluginName string) bool {
	// CSIMigration feature should be enabled along with the plugin-specific one
	// CSIMigration has been GA. It will be enabled by default.

	switch pluginName {
	case csilibplugins.AWSEBSInTreePluginName:
		return true
	case csilibplugins.GCEPDInTreePluginName:
		return pm.featureGate.Enabled(features.CSIMigrationGCE)
	case csilibplugins.AzureFileInTreePluginName:
		return pm.featureGate.Enabled(features.CSIMigrationAzureFile)
	case csilibplugins.AzureDiskInTreePluginName:
		return true
	case csilibplugins.CinderInTreePluginName:
		return true
	case csilibplugins.VSphereInTreePluginName:
		return pm.featureGate.Enabled(features.CSIMigrationvSphere)
	case csilibplugins.PortworxVolumePluginName:
		return pm.featureGate.Enabled(features.CSIMigrationPortworx)
	case csilibplugins.RBDVolumePluginName:
		return pm.featureGate.Enabled(features.CSIMigrationRBD)
	default:
		return false
	}
}


// IsMigratable indicates whether CSI migration has been enabled for a volume
// plugin that the spec refers to
func (pm PluginManager) IsMigratable(spec *volume.Spec) (bool, error) {
	if spec == nil {
		return false, fmt.Errorf("could not find if plugin is migratable because volume spec is nil")
	}

	pluginName, _ := pm.GetInTreePluginNameFromSpec(spec.PersistentVolume, spec.Volume)
	if pluginName == "" {
		return false, nil
	}
	// found an in-tree plugin that supports the spec
	return pm.IsMigrationEnabledForPlugin(pluginName), nil
}


// TranslateInTreeSpecToCSI translates a volume spec (either PV or inline volume)
// supported by an in-tree plugin to CSI
func TranslateInTreeSpecToCSI(spec *volume.Spec, podNamespace string, translator InTreeToCSITranslator) (*volume.Spec, error) {
	var csiPV *v1.PersistentVolume
	var err error
	inlineVolume := false
	if spec.PersistentVolume != nil {
		csiPV, err = translator.TranslateInTreePVToCSI(spec.PersistentVolume)
	} else if spec.Volume != nil {
		csiPV, err = translator.TranslateInTreeInlineVolumeToCSI(spec.Volume, podNamespace)
		inlineVolume = true
	} else {
		err = errors.New("not a valid volume spec")
	}
	if err != nil {
		return nil, fmt.Errorf("failed to translate in-tree pv to CSI: %v", err)
	}
	return &volume.Spec{
		Migrated:                        true,
		PersistentVolume:                csiPV,
		ReadOnly:                        spec.ReadOnly,
		InlineVolumeSpecForCSIMigration: inlineVolume,
	}, nil
}


// CheckMigrationFeatureFlags checks the configuration of feature flags related
// to CSI Migration is valid. It will return whether the migration is complete
// by looking up the pluginUnregister flag
func CheckMigrationFeatureFlags(f featuregate.FeatureGate, pluginMigration,
	pluginUnregister featuregate.Feature) (migrationComplete bool, err error) {
	// This is for in-tree plugin that get migration finished
	if f.Enabled(pluginMigration) && f.Enabled(pluginUnregister) {
		return true, nil
	}
	return false, nil
}


// ProbeVolumePlugins is the entry point for plugin detection in a package.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&downwardAPIPlugin{}}
}


func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
	return host.GetPodVolumeDir(uid, utilstrings.EscapeQualifiedName(downwardAPIPluginName), volName)
}


func wrappedVolumeSpec() volume.Spec

func wrappedVolumeSpec() volume.Spec {
	return volume.Spec{
		Volume: &v1.Volume{VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{Medium: v1.StorageMediumMemory}}},
	}
}


func (plugin *downwardAPIPlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	return nil
}


func (plugin *downwardAPIPlugin) GetPluginName() string {
	return downwardAPIPluginName
}


func (plugin *downwardAPIPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	volumeSource, _ := getVolumeSource(spec)
	if volumeSource == nil {
		return "", fmt.Errorf("Spec does not reference a DownwardAPI volume type")
	}

	// Return user defined volume name, since this is an ephemeral volume type
	return spec.Name(), nil
}


func (plugin *downwardAPIPlugin) CanSupport(spec *volume.Spec) bool {
	return spec.Volume != nil && spec.Volume.DownwardAPI != nil
}


func (plugin *downwardAPIPlugin) RequiresRemount(spec *volume.Spec) bool {
	return true
}


func (plugin *downwardAPIPlugin) SupportsMountOption() bool {
	return false
}


func (plugin *downwardAPIPlugin) SupportsBulkVolumeVerification() bool {
	return false
}


func (plugin *downwardAPIPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return false, nil
}


func (plugin *downwardAPIPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) {
	v := &downwardAPIVolume{
		volName:         spec.Name(),
		items:           spec.Volume.DownwardAPI.Items,
		pod:             pod,
		podUID:          pod.UID,
		plugin:          plugin,
		MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsDu(getPath(pod.UID, spec.Name(), plugin.host))),
	}
	return &downwardAPIVolumeMounter{
		downwardAPIVolume: v,
		source:            *spec.Volume.DownwardAPI,
		opts:              &opts,
	}, nil
}


func (plugin *downwardAPIPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	return &downwardAPIVolumeUnmounter{
		&downwardAPIVolume{
			volName:         volName,
			podUID:          podUID,
			plugin:          plugin,
			MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsDu(getPath(podUID, volName, plugin.host))),
		},
	}, nil
}


func (plugin *downwardAPIPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
	downwardAPIVolume := &v1.Volume{
		Name: volumeName,
		VolumeSource: v1.VolumeSource{
			DownwardAPI: &v1.DownwardAPIVolumeSource{},
		},
	}
	return volume.ReconstructedVolume{
		Spec: volume.NewSpecFromVolume(downwardAPIVolume),
	}, nil
}


// downward API volumes are always ReadOnlyManaged
func (d *downwardAPIVolume) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:       true,
		Managed:        true,
		SELinuxRelabel: true,
	}
}


// SetUp puts in place the volume plugin.
// This function is not idempotent by design. We want the data to be refreshed periodically.
// The internal sync interval of kubelet will drive the refresh of data.
// TODO: Add volume specific ticker and refresh loop
func (b *downwardAPIVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error {
	return b.SetUpAt(b.GetPath(), mounterArgs)
}


func (b *downwardAPIVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	klog.V(3).Infof("Setting up a downwardAPI volume %v for pod %v/%v at %v", b.volName, b.pod.Namespace, b.pod.Name, dir)
	// Wrap EmptyDir. Here we rely on the idempotency of the wrapped plugin to avoid repeatedly mounting
	wrapped, err := b.plugin.host.NewWrapperMounter(b.volName, wrappedVolumeSpec(), b.pod, *b.opts)
	if err != nil {
		klog.Errorf("Couldn't setup downwardAPI volume %v for pod %v/%v: %s", b.volName, b.pod.Namespace, b.pod.Name, err.Error())
		return err
	}

	data, err := CollectData(b.source.Items, b.pod, b.plugin.host, b.source.DefaultMode)
	if err != nil {
		klog.Errorf("Error preparing data for downwardAPI volume %v for pod %v/%v: %s", b.volName, b.pod.Namespace, b.pod.Name, err.Error())
		return err
	}

	setupSuccess := false
	if err := wrapped.SetUpAt(dir, mounterArgs); err != nil {
		klog.Errorf("Unable to setup downwardAPI volume %v for pod %v/%v: %s", b.volName, b.pod.Namespace, b.pod.Name, err.Error())
		return err
	}

	if err := volumeutil.MakeNestedMountpoints(b.volName, dir, *b.pod); err != nil {
		return err
	}

	defer func() {
		// Clean up directories if setup fails
		if !setupSuccess {
			unmounter, unmountCreateErr := b.plugin.NewUnmounter(b.volName, b.podUID)
			if unmountCreateErr != nil {
				klog.Errorf("error cleaning up mount %s after failure. Create unmounter failed with %v", b.volName, unmountCreateErr)
				return
			}
			tearDownErr := unmounter.TearDown()
			if tearDownErr != nil {
				klog.Errorf("error tearing down volume %s with : %v", b.volName, tearDownErr)
			}
		}
	}()

	writerContext := fmt.Sprintf("pod %v/%v volume %v", b.pod.Namespace, b.pod.Name, b.volName)
	writer, err := volumeutil.NewAtomicWriter(dir, writerContext)
	if err != nil {
		klog.Errorf("Error creating atomic writer: %v", err)
		return err
	}

	setPerms := func(_ string) error {
		// This may be the first time writing and new files get created outside the timestamp subdirectory:
		// change the permissions on the whole volume and not only in the timestamp directory.
		return volume.SetVolumeOwnership(b, dir, mounterArgs.FsGroup, nil /*fsGroupChangePolicy*/, volumeutil.FSGroupCompleteHook(b.plugin, nil))
	}
	err = writer.Write(data, setPerms)
	if err != nil {
		klog.Errorf("Error writing payload to dir: %v", err)
		return err
	}

	setupSuccess = true
	return nil
}


// CollectData collects requested downwardAPI in data map.
// Map's key is the requested name of file to dump
// Map's value is the (sorted) content of the field to be dumped in the file.
//
// Note: this function is exported so that it can be called from the projection volume driver
func CollectData(items []v1.DownwardAPIVolumeFile, pod *v1.Pod, host volume.VolumeHost, defaultMode *int32) (map[string]volumeutil.FileProjection, error) {
	if defaultMode == nil {
		return nil, fmt.Errorf("no defaultMode used, not even the default value for it")
	}

	errlist := []error{}
	data := make(map[string]volumeutil.FileProjection)
	for _, fileInfo := range items {
		var fileProjection volumeutil.FileProjection
		fPath := filepath.Clean(fileInfo.Path)
		if fileInfo.Mode != nil {
			fileProjection.Mode = *fileInfo.Mode
		} else {
			fileProjection.Mode = *defaultMode
		}
		if fileInfo.FieldRef != nil {
			// TODO: unify with Kubelet.podFieldSelectorRuntimeValue
			if values, err := fieldpath.ExtractFieldPathAsString(pod, fileInfo.FieldRef.FieldPath); err != nil {
				klog.Errorf("Unable to extract field %s: %s", fileInfo.FieldRef.FieldPath, err.Error())
				errlist = append(errlist, err)
			} else {
				fileProjection.Data = []byte(values)
			}
		} else if fileInfo.ResourceFieldRef != nil {
			containerName := fileInfo.ResourceFieldRef.ContainerName
			nodeAllocatable, err := host.GetNodeAllocatable()
			if err != nil {
				errlist = append(errlist, err)
			} else if values, err := resource.ExtractResourceValueByContainerNameAndNodeAllocatable(fileInfo.ResourceFieldRef, pod, containerName, nodeAllocatable); err != nil {
				klog.Errorf("Unable to extract field %s: %s", fileInfo.ResourceFieldRef.Resource, err.Error())
				errlist = append(errlist, err)
			} else {
				fileProjection.Data = []byte(values)
			}
		}

		data[fPath] = fileProjection
	}
	return data, utilerrors.NewAggregate(errlist)
}


func (d *downwardAPIVolume) GetPath() string {
	return d.plugin.host.GetPodVolumeDir(d.podUID, utilstrings.EscapeQualifiedName(downwardAPIPluginName), d.volName)
}


func (c *downwardAPIVolumeUnmounter) TearDown() error {
	return c.TearDownAt(c.GetPath())
}


func (c *downwardAPIVolumeUnmounter) TearDownAt(dir string) error {
	return volumeutil.UnmountViaEmptyDir(dir, c.plugin.host, c.volName, wrappedVolumeSpec(), c.podUID)
}


func getVolumeSource(spec *volume.Spec) (*v1.DownwardAPIVolumeSource, bool) {
	var readOnly bool
	var volumeSource *v1.DownwardAPIVolumeSource

	if spec.Volume != nil && spec.Volume.DownwardAPI != nil {
		volumeSource = spec.Volume.DownwardAPI
		readOnly = spec.ReadOnly
	}

	return volumeSource, readOnly
}


// ProbeVolumePlugins is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{
		&emptyDirPlugin{nil},
	}
}


func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
	return host.GetPodVolumeDir(uid, utilstrings.EscapeQualifiedName(emptyDirPluginName), volName)
}


func (plugin *emptyDirPlugin) Init(host volume.VolumeHost) error {
	plugin.host = host

	return nil
}


func (plugin *emptyDirPlugin) GetPluginName() string {
	return emptyDirPluginName
}


func (plugin *emptyDirPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	volumeSource, _ := getVolumeSource(spec)
	if volumeSource == nil {
		return "", fmt.Errorf("Spec does not reference an EmptyDir volume type")
	}

	// Return user defined volume name, since this is an ephemeral volume type
	return spec.Name(), nil
}


func (plugin *emptyDirPlugin) CanSupport(spec *volume.Spec) bool {
	return spec.Volume != nil && spec.Volume.EmptyDir != nil
}


func (plugin *emptyDirPlugin) RequiresRemount(spec *volume.Spec) bool {
	return false
}


func (plugin *emptyDirPlugin) SupportsMountOption() bool {
	return false
}


func (plugin *emptyDirPlugin) SupportsBulkVolumeVerification() bool {
	return false
}


func (plugin *emptyDirPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return false, nil
}


func (plugin *emptyDirPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) {
	return plugin.newMounterInternal(spec, pod, plugin.host.GetMounter(plugin.GetPluginName()), &realMountDetector{plugin.host.GetMounter(plugin.GetPluginName())}, opts)
}


func calculateEmptyDirMemorySize(nodeAllocatableMemory *resource.Quantity, spec *volume.Spec, pod *v1.Pod) *resource.Quantity {
	// if feature is disabled, continue the default behavior of linux host default
	sizeLimit := &resource.Quantity{}
	if !utilfeature.DefaultFeatureGate.Enabled(features.SizeMemoryBackedVolumes) {
		return sizeLimit
	}

	// size limit defaults to node allocatable (pods can't consume more memory than all pods)
	sizeLimit = nodeAllocatableMemory
	zero := resource.MustParse("0")

	// determine pod resource allocation
	// we use the same function for pod cgroup assignment to maintain consistent behavior
	// NOTE: this could be nil on systems that do not support pod memory containment (i.e. windows)
	podResourceConfig := cm.ResourceConfigForPod(pod, false, uint64(100000), false)
	if podResourceConfig != nil && podResourceConfig.Memory != nil {
		podMemoryLimit := resource.NewQuantity(*(podResourceConfig.Memory), resource.BinarySI)
		// ensure 0 < value < size
		if podMemoryLimit.Cmp(zero) > 0 && podMemoryLimit.Cmp(*sizeLimit) < 1 {
			sizeLimit = podMemoryLimit
		}
	}

	// volume local size is  used if and only if less than what pod could consume
	if spec.Volume.EmptyDir.SizeLimit != nil {
		volumeSizeLimit := spec.Volume.EmptyDir.SizeLimit
		// ensure 0 < value < size
		if volumeSizeLimit.Cmp(zero) > 0 && volumeSizeLimit.Cmp(*sizeLimit) < 1 {
			sizeLimit = volumeSizeLimit
		}
	}
	return sizeLimit
}


func (plugin *emptyDirPlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod, mounter mount.Interface, mountDetector mountDetector, opts volume.VolumeOptions) (volume.Mounter, error) {
	medium := v1.StorageMediumDefault
	sizeLimit := &resource.Quantity{}
	if spec.Volume.EmptyDir != nil { // Support a non-specified source as EmptyDir.
		medium = spec.Volume.EmptyDir.Medium
		if medium == v1.StorageMediumMemory {
			nodeAllocatable, err := plugin.host.GetNodeAllocatable()
			if err != nil {
				return nil, err
			}
			sizeLimit = calculateEmptyDirMemorySize(nodeAllocatable.Memory(), spec, pod)
		}
	}
	return &emptyDir{
		pod:             pod,
		volName:         spec.Name(),
		medium:          medium,
		sizeLimit:       sizeLimit,
		mounter:         mounter,
		mountDetector:   mountDetector,
		plugin:          plugin,
		MetricsProvider: volume.NewMetricsDu(getPath(pod.UID, spec.Name(), plugin.host)),
	}, nil
}


func (plugin *emptyDirPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	// Inject real implementations here, test through the internal function.
	return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter(plugin.GetPluginName()), &realMountDetector{plugin.host.GetMounter(plugin.GetPluginName())})
}


func (plugin *emptyDirPlugin) newUnmounterInternal(volName string, podUID types.UID, mounter mount.Interface, mountDetector mountDetector) (volume.Unmounter, error) {
	ed := &emptyDir{
		pod:             &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: podUID}},
		volName:         volName,
		medium:          v1.StorageMediumDefault, // might be changed later
		mounter:         mounter,
		mountDetector:   mountDetector,
		plugin:          plugin,
		MetricsProvider: volume.NewMetricsDu(getPath(podUID, volName, plugin.host)),
	}
	return ed, nil
}


func (plugin *emptyDirPlugin) ConstructVolumeSpec(volName, mountPath string) (volume.ReconstructedVolume, error) {
	emptyDirVolume := &v1.Volume{
		Name: volName,
		VolumeSource: v1.VolumeSource{
			EmptyDir: &v1.EmptyDirVolumeSource{},
		},
	}
	return volume.ReconstructedVolume{
		Spec: volume.NewSpecFromVolume(emptyDirVolume),
	}, nil
}


func (ed *emptyDir) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:       false,
		Managed:        true,
		SELinuxRelabel: true,
	}
}


// SetUp creates new directory.
func (ed *emptyDir) SetUp(mounterArgs volume.MounterArgs) error {
	return ed.SetUpAt(ed.GetPath(), mounterArgs)
}


// SetUpAt creates new directory.
func (ed *emptyDir) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	notMnt, err := ed.mounter.IsLikelyNotMountPoint(dir)
	// Getting an os.IsNotExist err from is a contingency; the directory
	// may not exist yet, in which case, setup should run.
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	// If the plugin readiness file is present for this volume, and the
	// storage medium is the default, then the volume is ready.  If the
	// medium is memory, and a mountpoint is present, then the volume is
	// ready.
	readyDir := ed.getMetaDir()
	if volumeutil.IsReady(readyDir) {
		if ed.medium == v1.StorageMediumMemory && !notMnt {
			return nil
		} else if ed.medium == v1.StorageMediumDefault {
			// Further check dir exists
			if _, err := os.Stat(dir); err == nil {
				klog.V(6).InfoS("Dir exists, so check and assign quota if the underlying medium supports quotas", "dir", dir)
				err = ed.assignQuota(dir, mounterArgs.DesiredSize)
				return err
			}
			// This situation should not happen unless user manually delete volume dir.
			// In this case, delete ready file and print a warning for it.
			klog.Warningf("volume ready file dir %s exist, but volume dir %s does not. Remove ready dir", readyDir, dir)
			if err := os.RemoveAll(readyDir); err != nil && !os.IsNotExist(err) {
				klog.Warningf("failed to remove ready dir [%s]: %v", readyDir, err)
			}
		}
	}

	switch {
	case ed.medium == v1.StorageMediumDefault:
		err = ed.setupDir(dir)
	case ed.medium == v1.StorageMediumMemory:
		err = ed.setupTmpfs(dir)
	case v1helper.IsHugePageMedium(ed.medium):
		err = ed.setupHugepages(dir)
	default:
		err = fmt.Errorf("unknown storage medium %q", ed.medium)
	}

	volume.SetVolumeOwnership(ed, dir, mounterArgs.FsGroup, nil /*fsGroupChangePolicy*/, volumeutil.FSGroupCompleteHook(ed.plugin, nil))

	// If setting up the quota fails, just log a message but don't actually error out.
	// We'll use the old du mechanism in this case, at least until we support
	// enforcement.
	if err == nil {
		volumeutil.SetReady(ed.getMetaDir())
		err = ed.assignQuota(dir, mounterArgs.DesiredSize)
	}
	return err
}


// assignQuota checks if the underlying medium supports quotas and if so, sets
func (ed *emptyDir) assignQuota(dir string, mounterSize *resource.Quantity) error {
	if mounterSize != nil {
		// Deliberately shadow the outer use of err as noted
		// above.
		hasQuotas, err := fsquota.SupportsQuotas(ed.mounter, dir)
		if err != nil {
			klog.V(3).Infof("Unable to check for quota support on %s: %s", dir, err.Error())
		} else if hasQuotas {
			klog.V(4).Infof("emptydir trying to assign quota %v on %s", mounterSize, dir)
			if err := fsquota.AssignQuota(ed.mounter, dir, ed.pod.UID, mounterSize); err != nil {
				klog.V(3).Infof("Set quota on %s failed %s", dir, err.Error())
				return err
			}
			return nil
		}
	}
	return nil
}


// setupTmpfs creates a tmpfs mount at the specified directory.
func (ed *emptyDir) setupTmpfs(dir string) error {
	if ed.mounter == nil {
		return fmt.Errorf("memory storage requested, but mounter is nil")
	}
	if err := ed.setupDir(dir); err != nil {
		return err
	}
	// Make SetUp idempotent.
	medium, isMnt, _, err := ed.mountDetector.GetMountMedium(dir, ed.medium)
	if err != nil {
		return err
	}
	// If the directory is a mountpoint with medium memory, there is no
	// work to do since we are already in the desired state.
	if isMnt && medium == v1.StorageMediumMemory {
		return nil
	}

	var options []string
	// Linux system default is 50% of capacity.
	if ed.sizeLimit != nil && ed.sizeLimit.Value() > 0 {
		options = []string{fmt.Sprintf("size=%d", ed.sizeLimit.Value())}
	}

	klog.V(3).Infof("pod %v: mounting tmpfs for volume %v", ed.pod.UID, ed.volName)
	return ed.mounter.MountSensitiveWithoutSystemd("tmpfs", dir, "tmpfs", options, nil)
}


// setupHugepages creates a hugepage mount at the specified directory.
func (ed *emptyDir) setupHugepages(dir string) error {
	if ed.mounter == nil {
		return fmt.Errorf("memory storage requested, but mounter is nil")
	}
	if err := ed.setupDir(dir); err != nil {
		return err
	}
	// Make SetUp idempotent.
	medium, isMnt, mountPageSize, err := ed.mountDetector.GetMountMedium(dir, ed.medium)
	klog.V(3).Infof("pod %v: setupHugepages: medium: %s, isMnt: %v, dir: %s, err: %v", ed.pod.UID, medium, isMnt, dir, err)
	if err != nil {
		return err
	}
	// If the directory is a mountpoint with medium hugepages of the same page size,
	// there is no work to do since we are already in the desired state.
	if isMnt && v1helper.IsHugePageMedium(medium) {
		// Medium is: Hugepages
		if ed.medium == v1.StorageMediumHugePages {
			return nil
		}
		if mountPageSize == nil {
			return fmt.Errorf("pod %v: mounted dir %s pagesize is not determined", ed.pod.UID, dir)
		}
		// Medium is: Hugepages-<size>
		// Mounted page size and medium size must be equal
		mediumSize, err := v1helper.HugePageSizeFromMedium(ed.medium)
		if err != nil {
			return err
		}
		if mountPageSize == nil || mediumSize.Cmp(*mountPageSize) != 0 {
			return fmt.Errorf("pod %v: mounted dir %s pagesize '%s' and requested medium size '%s' differ", ed.pod.UID, dir, mountPageSize.String(), mediumSize.String())
		}
		return nil
	}

	pageSizeMountOption, err := getPageSizeMountOption(ed.medium, ed.pod)
	if err != nil {
		return err
	}

	klog.V(3).Infof("pod %v: mounting hugepages for volume %v", ed.pod.UID, ed.volName)
	return ed.mounter.MountSensitiveWithoutSystemd("nodev", dir, "hugetlbfs", []string{pageSizeMountOption}, nil)
}


// getPageSizeMountOption retrieves pageSize mount option from Pod's resources
// and medium and validates pageSize options in all containers of given Pod.
func getPageSizeMountOption(medium v1.StorageMedium, pod *v1.Pod) (string, error) {
	pageSizeFound := false
	pageSize := resource.Quantity{}

	var mediumPageSize resource.Quantity
	if medium != v1.StorageMediumHugePages {
		// medium is: Hugepages-<size>
		var err error
		mediumPageSize, err = v1helper.HugePageSizeFromMedium(medium)
		if err != nil {
			return "", err
		}
	}

	// In some rare cases init containers can also consume Huge pages
	for _, container := range append(pod.Spec.Containers, pod.Spec.InitContainers...) {
		// We can take request because limit and requests must match.
		for requestName := range container.Resources.Requests {
			if !v1helper.IsHugePageResourceName(requestName) {
				continue
			}
			currentPageSize, err := v1helper.HugePageSizeFromResourceName(requestName)
			if err != nil {
				return "", err
			}
			if medium == v1.StorageMediumHugePages { // medium is: Hugepages, size is not specified
				// PageSize for all volumes in a POD must be equal if medium is "Hugepages"
				if pageSizeFound && pageSize.Cmp(currentPageSize) != 0 {
					return "", fmt.Errorf("medium: %s can't be used if container requests multiple huge page sizes", medium)
				}

				pageSizeFound = true
				pageSize = currentPageSize
			} else { // medium is: Hugepages-<size>
				if currentPageSize.Cmp(mediumPageSize) == 0 {
					pageSizeFound = true
					pageSize = currentPageSize
				}
			}
		}
	}

	if !pageSizeFound {
		return "", fmt.Errorf("medium %s: hugePages storage requested, but there is no resource request for huge pages", medium)
	}

	return fmt.Sprintf("%s=%s", hugePagesPageSizeMountOption, pageSize.String()), nil

}


// setupDir creates the directory with the default permissions specified by the perm constant.
func (ed *emptyDir) setupDir(dir string) error {
	// Create the directory if it doesn't already exist.
	if err := os.MkdirAll(dir, perm); err != nil {
		return err
	}

	// stat the directory to read permission bits
	fileinfo, err := os.Lstat(dir)
	if err != nil {
		return err
	}

	if fileinfo.Mode().Perm() != perm.Perm() {
		// If the permissions on the created directory are wrong, the
		// kubelet is probably running with a umask set.  In order to
		// avoid clearing the umask for the entire process or locking
		// the thread, clearing the umask, creating the dir, restoring
		// the umask, and unlocking the thread, we do a chmod to set
		// the specific bits we need.
		err := os.Chmod(dir, perm)
		if err != nil {
			return err
		}

		fileinfo, err = os.Lstat(dir)
		if err != nil {
			return err
		}

		if fileinfo.Mode().Perm() != perm.Perm() {
			klog.Errorf("Expected directory %q permissions to be: %s; got: %s", dir, perm.Perm(), fileinfo.Mode().Perm())
		}
	}

	return nil
}


func (ed *emptyDir) GetPath() string

func (ed *emptyDir) GetPath() string {
	return getPath(ed.pod.UID, ed.volName, ed.plugin.host)
}


func (ed *emptyDir) TearDown() error

// TearDown simply discards everything in the directory.
func (ed *emptyDir) TearDown() error {
	return ed.TearDownAt(ed.GetPath())
}


// TearDownAt simply discards everything in the directory.
func (ed *emptyDir) TearDownAt(dir string) error {
	// First remove ready dir which created in SetUp func
	readyDir := ed.getMetaDir()
	if removeErr := os.RemoveAll(readyDir); removeErr != nil && !os.IsNotExist(removeErr) {
		return fmt.Errorf("failed to remove ready dir [%s]: %v", readyDir, removeErr)
	}

	if pathExists, pathErr := mount.PathExists(dir); pathErr != nil {
		return fmt.Errorf("error checking if path exists: %w", pathErr)
	} else if !pathExists {
		klog.Warningf("Warning: Unmount skipped because path does not exist: %v", dir)
		return nil
	}

	// Figure out the medium.
	medium, isMnt, _, err := ed.mountDetector.GetMountMedium(dir, ed.medium)
	if err != nil {
		return err
	}
	if isMnt {
		if medium == v1.StorageMediumMemory {
			ed.medium = v1.StorageMediumMemory
			return ed.teardownTmpfsOrHugetlbfs(dir)
		} else if medium == v1.StorageMediumHugePages {
			ed.medium = v1.StorageMediumHugePages
			return ed.teardownTmpfsOrHugetlbfs(dir)
		}
	}
	// assume StorageMediumDefault
	return ed.teardownDefault(dir)
}


func (ed *emptyDir) teardownDefault(dir string) error {
	// Remove any quota
	err := fsquota.ClearQuota(ed.mounter, dir)
	if err != nil {
		klog.Warningf("Warning: Failed to clear quota on %s: %v", dir, err)
	}
	// Renaming the directory is not required anymore because the operation executor
	// now handles duplicate operations on the same volume
	return os.RemoveAll(dir)
}


func (ed *emptyDir) teardownTmpfsOrHugetlbfs(dir string) error {
	if ed.mounter == nil {
		return fmt.Errorf("memory storage requested, but mounter is nil")
	}
	if err := ed.mounter.Unmount(dir); err != nil {
		return err
	}
	if err := os.RemoveAll(dir); err != nil {
		return err
	}
	return nil
}


func (ed *emptyDir) getMetaDir() string

func (ed *emptyDir) getMetaDir() string {
	return filepath.Join(ed.plugin.host.GetPodPluginDir(ed.pod.UID, utilstrings.EscapeQualifiedName(emptyDirPluginName)), ed.volName)
}


func getVolumeSource(spec *volume.Spec) (*v1.EmptyDirVolumeSource, bool) {
	var readOnly bool
	var volumeSource *v1.EmptyDirVolumeSource

	if spec.Volume != nil && spec.Volume.EmptyDir != nil {
		volumeSource = spec.Volume.EmptyDir
		readOnly = spec.ReadOnly
	}

	return volumeSource, readOnly
}


// getPageSize obtains page size from the 'pagesize' mount option of the
// mounted volume
func getPageSize(path string, mounter mount.Interface) (*resource.Quantity, error) {
	// Get mount point data for the path
	mountPoints, err := mounter.List()
	if err != nil {
		return nil, fmt.Errorf("error listing mount points: %v", err)
	}
	// Find mount point for the path
	mountPoint, err := func(mps []mount.MountPoint, mpPath string) (*mount.MountPoint, error) {
		for _, mp := range mps {
			if mp.Path == mpPath {
				return &mp, nil
			}
		}
		return nil, fmt.Errorf("mount point for %s not found", mpPath)
	}(mountPoints, path)
	if err != nil {
		return nil, err
	}
	// Get page size from the 'pagesize' option value
	for _, opt := range mountPoint.Opts {
		opt = strings.TrimSpace(opt)
		prefix := "pagesize="
		if strings.HasPrefix(opt, prefix) {
			// NOTE: Adding suffix 'i' as result should be comparable with a medium size.
			// pagesize mount option is specified without a suffix,
			// e.g. pagesize=2M or pagesize=1024M for x86 CPUs
			trimmedOpt := strings.TrimPrefix(opt, prefix)
			if !strings.HasSuffix(trimmedOpt, "i") {
				trimmedOpt = trimmedOpt + "i"
			}
			pageSize, err := resource.ParseQuantity(trimmedOpt)
			if err != nil {
				return nil, fmt.Errorf("error getting page size from '%s' mount option: %v", opt, err)
			}
			return &pageSize, nil
		}
	}
	return nil, fmt.Errorf("no pagesize option specified for %s mount", mountPoint.Path)
}


func (m *realMountDetector) GetMountMedium(path string, requestedMedium v1.StorageMedium) (v1.StorageMedium, bool, *resource.Quantity, error) {
	klog.V(5).Infof("Determining mount medium of %v", path)
	notMnt, err := m.mounter.IsLikelyNotMountPoint(path)
	if err != nil {
		return v1.StorageMediumDefault, false, nil, fmt.Errorf("IsLikelyNotMountPoint(%q): %v", path, err)
	}

	buf := unix.Statfs_t{}
	if err := unix.Statfs(path, &buf); err != nil {
		return v1.StorageMediumDefault, false, nil, fmt.Errorf("statfs(%q): %v", path, err)
	}

	klog.V(3).Infof("Statfs_t of %v: %+v", path, buf)

	if buf.Type == linuxTmpfsMagic {
		return v1.StorageMediumMemory, !notMnt, nil, nil
	} else if int64(buf.Type) == linuxHugetlbfsMagic {
		// Skip page size detection if requested medium doesn't have size specified
		if requestedMedium == v1.StorageMediumHugePages {
			return v1.StorageMediumHugePages, !notMnt, nil, nil
		}
		// Get page size for the volume mount
		pageSize, err := getPageSize(path, m.mounter)
		if err != nil {
			return v1.StorageMediumHugePages, !notMnt, nil, err
		}
		return v1.StorageMediumHugePages, !notMnt, pageSize, nil
	}
	return v1.StorageMediumDefault, !notMnt, nil, nil
}


func (m *realMountDetector) GetMountMedium(path string, requestedMedium v1.StorageMedium) (v1.StorageMedium, bool, *resource.Quantity, error) {
	return v1.StorageMediumDefault, false, nil, nil
}


func (plugin *fcPlugin) NewAttacher() (volume.Attacher, error) {
	return &fcAttacher{
		host:    plugin.host,
		manager: &fcUtil{},
	}, nil
}


func (plugin *fcPlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
	return plugin.NewAttacher()
}


func (plugin *fcPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
	mounter := plugin.host.GetMounter(plugin.GetPluginName())
	return mounter.GetMountRefs(deviceMountPath)
}


func (attacher *fcAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) {
	return "", nil
}


func (attacher *fcAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.NodeName) (map[*volume.Spec]bool, error) {
	volumesAttachedCheck := make(map[*volume.Spec]bool)
	for _, spec := range specs {
		volumesAttachedCheck[spec] = true
	}

	return volumesAttachedCheck, nil
}


func (attacher *fcAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) {
	mounter, err := volumeSpecToMounter(spec, attacher.host)
	if err != nil {
		klog.Warningf("failed to get fc mounter: %v", err)
		return "", err
	}
	return attacher.manager.AttachDisk(*mounter)
}


func (attacher *fcAttacher) GetDeviceMountPath(
	spec *volume.Spec) (string, error) {
	mounter, err := volumeSpecToMounter(spec, attacher.host)
	if err != nil {
		klog.Warningf("failed to get fc mounter: %v", err)
		return "", err
	}

	return attacher.manager.MakeGlobalPDName(*mounter.fcDisk), nil
}


func (attacher *fcAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string, mountArgs volume.DeviceMounterArgs) error {
	mounter := attacher.host.GetMounter(fcPluginName)
	notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath)
	if err != nil {
		if os.IsNotExist(err) {
			if err := os.MkdirAll(deviceMountPath, 0750); err != nil {
				return err
			}
			notMnt = true
		} else {
			return err
		}
	}

	volumeSource, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return err
	}

	options := []string{}
	if readOnly {
		options = append(options, "ro")
	}
	if mountArgs.SELinuxLabel != "" {
		options = volumeutil.AddSELinuxMountOption(options, mountArgs.SELinuxLabel)
	}
	if notMnt {
		diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Exec: attacher.host.GetExec(fcPluginName)}
		mountOptions := volumeutil.MountOptionFromSpec(spec, options...)
		err = diskMounter.FormatAndMount(devicePath, deviceMountPath, volumeSource.FSType, mountOptions)
		if err != nil {
			os.Remove(deviceMountPath)
			return err
		}
	}
	return nil
}


func (plugin *fcPlugin) NewDetacher() (volume.Detacher, error) {
	return &fcDetacher{
		mounter: plugin.host.GetMounter(plugin.GetPluginName()),
		manager: &fcUtil{},
		host:    plugin.host,
	}, nil
}


func (plugin *fcPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
	return plugin.NewDetacher()
}


func (detacher *fcDetacher) Detach(volumeName string, nodeName types.NodeName) error {
	return nil
}


func (detacher *fcDetacher) UnmountDevice(deviceMountPath string) error {
	// Specify device name for DetachDisk later
	devName, _, err := mount.GetDeviceNameFromMount(detacher.mounter, deviceMountPath)
	if err != nil {
		klog.Errorf("fc: failed to get device from mnt: %s\nError: %v", deviceMountPath, err)
		return err
	}
	// Unmount for deviceMountPath(=globalPDPath)
	err = mount.CleanupMountPoint(deviceMountPath, detacher.mounter, false)
	if err != nil {
		return fmt.Errorf("fc: failed to unmount: %s\nError: %v", deviceMountPath, err)
	}
	// GetDeviceNameFromMount from above returns an empty string if deviceMountPath is not a mount point
	// There is no need to DetachDisk if this is the case (and DetachDisk will throw an error if we attempt)
	if devName == "" {
		return nil
	}

	unMounter := volumeSpecToUnmounter(detacher.mounter, detacher.host)
	// The device is unmounted now. If UnmountDevice was retried, GetDeviceNameFromMount
	// won't find any mount and won't return DetachDisk below.
	// Therefore implement our own retry mechanism here.
	// E.g. DetachDisk sometimes fails to flush a multipath device with "device is busy" when it was
	// just unmounted.
	// 2 minutes should be enough within 6 minute force detach timeout.
	var detachError error
	err = wait.PollImmediate(10*time.Second, 2*time.Minute, func() (bool, error) {
		detachError = detacher.manager.DetachDisk(*unMounter, devName)
		if detachError != nil {
			klog.V(4).Infof("fc: failed to detach disk %s (%s): %v", devName, deviceMountPath, detachError)
			return false, nil
		}
		return true, nil
	})
	if err != nil {
		return fmt.Errorf("fc: failed to detach disk: %s: %v", devName, detachError)
	}

	klog.V(2).Infof("fc: successfully detached disk: %s", devName)
	return nil
}


func (plugin *fcPlugin) CanAttach(spec *volume.Spec) (bool, error) {
	return true, nil
}


func (plugin *fcPlugin) CanDeviceMount(spec *volume.Spec) (bool, error) {
	return true, nil
}


func volumeSpecToMounter(spec *volume.Spec, host volume.VolumeHost) (*fcDiskMounter, error) {
	fc, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}
	var lun string
	var wwids []string
	if fc.Lun != nil && len(fc.TargetWWNs) != 0 {
		lun = strconv.Itoa(int(*fc.Lun))
	} else if len(fc.WWIDs) != 0 {
		for _, wwid := range fc.WWIDs {
			wwids = append(wwids, strings.Replace(wwid, " ", "_", -1))
		}
	} else {
		return nil, fmt.Errorf("fc: no fc disk information found. failed to make a new mounter")
	}
	fcDisk := &fcDisk{
		plugin: &fcPlugin{
			host: host,
		},
		wwns:  fc.TargetWWNs,
		lun:   lun,
		wwids: wwids,
		io:    &osIOHandler{},
	}

	volumeMode, err := volumeutil.GetVolumeMode(spec)
	if err != nil {
		return nil, err
	}

	klog.V(5).Infof("fc: volumeSpecToMounter volumeMode %s", volumeMode)
	return &fcDiskMounter{
		fcDisk:       fcDisk,
		fsType:       fc.FSType,
		volumeMode:   volumeMode,
		readOnly:     readOnly,
		mounter:      volumeutil.NewSafeFormatAndMountFromHost(fcPluginName, host),
		deviceUtil:   volumeutil.NewDeviceHandler(volumeutil.NewIOHandler()),
		mountOptions: volumeutil.MountOptionFromSpec(spec),
	}, nil
}


func volumeSpecToUnmounter(mounter mount.Interface, host volume.VolumeHost) *fcDiskUnmounter {
	return &fcDiskUnmounter{
		fcDisk: &fcDisk{
			io: &osIOHandler{},
		},
		mounter:    mounter,
		deviceUtil: volumeutil.NewDeviceHandler(volumeutil.NewIOHandler()),
		exec:       host.GetExec(fcPluginName),
	}
}


// utility to mount a disk based filesystem
func diskSetUp(manager diskManager, b fcDiskMounter, volPath string, mounter mount.Interface, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy) error {
	globalPDPath := manager.MakeGlobalPDName(*b.fcDisk)
	noMnt, err := mounter.IsLikelyNotMountPoint(volPath)

	if err != nil && !os.IsNotExist(err) {
		klog.Errorf("cannot validate mountpoint: %s", volPath)
		return err
	}
	if !noMnt {
		return nil
	}
	if err := os.MkdirAll(volPath, 0750); err != nil {
		klog.Errorf("failed to mkdir:%s", volPath)
		return err
	}
	// Perform a bind mount to the full path to allow duplicate mounts of the same disk.
	options := []string{"bind"}
	if b.readOnly {
		options = append(options, "ro")
	}
	mountOptions := util.JoinMountOptions(options, b.mountOptions)
	err = mounter.MountSensitiveWithoutSystemd(globalPDPath, volPath, "", mountOptions, nil)
	if err != nil {
		klog.Errorf("Failed to bind mount: source:%s, target:%s, err:%v", globalPDPath, volPath, err)
		noMnt, mntErr := b.mounter.IsLikelyNotMountPoint(volPath)
		if mntErr != nil {
			klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
			return err
		}
		if !noMnt {
			if mntErr = b.mounter.Unmount(volPath); mntErr != nil {
				klog.Errorf("Failed to unmount: %v", mntErr)
				return err
			}
			noMnt, mntErr = b.mounter.IsLikelyNotMountPoint(volPath)
			if mntErr != nil {
				klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
				return err
			}
			if !noMnt {
				//  will most likely retry on next sync loop.
				klog.Errorf("%s is still mounted, despite call to unmount().  Will try again next sync loop.", volPath)
				return err
			}
		}
		os.Remove(volPath)

		return err
	}

	if !b.readOnly {
		volume.SetVolumeOwnership(&b, volPath, fsGroup, fsGroupChangePolicy, util.FSGroupCompleteHook(b.plugin, nil))
	}

	return nil
}


// ProbeVolumePlugins is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&fcPlugin{nil}}
}


func (plugin *fcPlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	return nil
}


func (plugin *fcPlugin) GetPluginName() string {
	return fcPluginName
}


func (plugin *fcPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	// API server validates these parameters beforehand but attach/detach
	// controller creates volumespec without validation. They may be nil
	// or zero length. We should check again to avoid unexpected conditions.
	if len(volumeSource.TargetWWNs) != 0 && volumeSource.Lun != nil {
		// TargetWWNs are the FibreChannel target worldwide names
		return fmt.Sprintf("%v:%v", volumeSource.TargetWWNs, *volumeSource.Lun), nil
	} else if len(volumeSource.WWIDs) != 0 {
		// WWIDs are the FibreChannel World Wide Identifiers
		return fmt.Sprintf("%v", volumeSource.WWIDs), nil
	}

	return "", err
}


func (plugin *fcPlugin) CanSupport(spec *volume.Spec) bool {
	return (spec.Volume != nil && spec.Volume.FC != nil) || (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.FC != nil)
}


func (plugin *fcPlugin) RequiresRemount(spec *volume.Spec) bool {
	return false
}


func (plugin *fcPlugin) SupportsMountOption() bool {
	return true
}


func (plugin *fcPlugin) SupportsBulkVolumeVerification() bool {
	return false
}


func (plugin *fcPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return true, nil
}


func (plugin *fcPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
	return []v1.PersistentVolumeAccessMode{
		v1.ReadWriteOnce,
		v1.ReadOnlyMany,
	}
}


func (plugin *fcPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
	// Inject real implementations here, test through the internal function.
	return plugin.newMounterInternal(spec, pod.UID, &fcUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()))
}


func (plugin *fcPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface, exec utilexec.Interface) (volume.Mounter, error) {
	// fc volumes used directly in a pod have a ReadOnly flag set by the pod author.
	// fc volumes used as a PersistentVolume gets the ReadOnly flag indirectly through the persistent-claim volume used to mount the PV
	fc, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}

	wwns, lun, wwids, err := getWwnsLunWwids(fc)
	if err != nil {
		return nil, fmt.Errorf("fc: no fc disk information found. failed to make a new mounter")
	}
	fcDisk := &fcDisk{
		podUID:  podUID,
		volName: spec.Name(),
		wwns:    wwns,
		lun:     lun,
		wwids:   wwids,
		manager: manager,
		io:      &osIOHandler{},
		plugin:  plugin,
	}

	volumeMode, err := util.GetVolumeMode(spec)
	if err != nil {
		return nil, err
	}

	klog.V(5).Infof("fc: newMounterInternal volumeMode %s", volumeMode)
	return &fcDiskMounter{
		fcDisk:       fcDisk,
		fsType:       fc.FSType,
		volumeMode:   volumeMode,
		readOnly:     readOnly,
		mounter:      &mount.SafeFormatAndMount{Interface: mounter, Exec: exec},
		deviceUtil:   util.NewDeviceHandler(util.NewIOHandler()),
		mountOptions: util.MountOptionFromSpec(spec),
	}, nil
}


func (plugin *fcPlugin) NewBlockVolumeMapper(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.BlockVolumeMapper, error) {
	// If this called via GenerateUnmapDeviceFunc(), pod is nil.
	// Pass empty string as dummy uid since uid isn't used in the case.
	var uid types.UID
	if pod != nil {
		uid = pod.UID
	}
	return plugin.newBlockVolumeMapperInternal(spec, uid, &fcUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()))
}


func (plugin *fcPlugin) newBlockVolumeMapperInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface, exec utilexec.Interface) (volume.BlockVolumeMapper, error) {
	fc, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}

	wwns, lun, wwids, err := getWwnsLunWwids(fc)
	if err != nil {
		return nil, fmt.Errorf("fc: no fc disk information found. failed to make a new mapper")
	}

	mapper := &fcDiskMapper{
		fcDisk: &fcDisk{
			podUID:  podUID,
			volName: spec.Name(),
			wwns:    wwns,
			lun:     lun,
			wwids:   wwids,
			manager: manager,
			io:      &osIOHandler{},
			plugin:  plugin},
		readOnly:   readOnly,
		mounter:    &mount.SafeFormatAndMount{Interface: mounter, Exec: exec},
		deviceUtil: util.NewDeviceHandler(util.NewIOHandler()),
	}

	blockPath, err := mapper.GetGlobalMapPath(spec)
	if err != nil {
		return nil, fmt.Errorf("failed to get device path: %v", err)
	}
	mapper.MetricsProvider = volume.NewMetricsBlock(filepath.Join(blockPath, string(podUID)))

	return mapper, nil
}


func (plugin *fcPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	// Inject real implementations here, test through the internal function.
	return plugin.newUnmounterInternal(volName, podUID, &fcUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()))
}


func (plugin *fcPlugin) newUnmounterInternal(volName string, podUID types.UID, manager diskManager, mounter mount.Interface, exec utilexec.Interface) (volume.Unmounter, error) {
	return &fcDiskUnmounter{
		fcDisk: &fcDisk{
			podUID:  podUID,
			volName: volName,
			manager: manager,
			plugin:  plugin,
			io:      &osIOHandler{},
		},
		mounter:    mounter,
		deviceUtil: util.NewDeviceHandler(util.NewIOHandler()),
		exec:       exec,
	}, nil
}


func (plugin *fcPlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (volume.BlockVolumeUnmapper, error) {
	return plugin.newUnmapperInternal(volName, podUID, &fcUtil{}, plugin.host.GetExec(plugin.GetPluginName()))
}


func (plugin *fcPlugin) newUnmapperInternal(volName string, podUID types.UID, manager diskManager, exec utilexec.Interface) (volume.BlockVolumeUnmapper, error) {
	return &fcDiskUnmapper{
		fcDisk: &fcDisk{
			podUID:  podUID,
			volName: volName,
			manager: manager,
			plugin:  plugin,
			io:      &osIOHandler{},
		},
		exec:       exec,
		deviceUtil: util.NewDeviceHandler(util.NewIOHandler()),
	}, nil
}


func (plugin *fcPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
	// Find globalPDPath from pod volume directory(mountPath)
	// examples:
	//   mountPath:     pods/{podUid}/volumes/kubernetes.io~fc/{volumeName}
	//   globalPDPath : plugins/kubernetes.io/fc/50060e801049cfd1-lun-0
	var globalPDPath string

	mounter := plugin.host.GetMounter(plugin.GetPluginName())
	// Try really hard to get the global mount of the volume, an error returned from here would
	// leave the global mount still mounted, while marking the volume as unused.
	// The volume can then be mounted on several nodes, resulting in volume
	// corruption.
	paths, err := util.GetReliableMountRefs(mounter, mountPath)
	if io.IsInconsistentReadError(err) {
		klog.Errorf("Failed to read mount refs from /proc/mounts for %s: %s", mountPath, err)
		klog.Errorf("Kubelet cannot unmount volume at %s, please unmount it manually", mountPath)
		return volume.ReconstructedVolume{}, err
	}
	if err != nil {
		return volume.ReconstructedVolume{}, err
	}
	for _, path := range paths {
		if strings.Contains(path, plugin.host.GetPluginDir(fcPluginName)) {
			globalPDPath = path
			break
		}
	}
	// Couldn't fetch globalPDPath
	if len(globalPDPath) == 0 {
		return volume.ReconstructedVolume{}, fmt.Errorf("couldn't fetch globalPDPath. failed to obtain volume spec")
	}

	wwns, lun, wwids, err := parsePDName(globalPDPath)
	if err != nil {
		return volume.ReconstructedVolume{}, fmt.Errorf("failed to retrieve volume plugin information from globalPDPath: %s", err)
	}
	// Create volume from wwn+lun or wwid
	fcVolume := &v1.Volume{
		Name: volumeName,
		VolumeSource: v1.VolumeSource{
			FC: &v1.FCVolumeSource{WWIDs: wwids, Lun: &lun, TargetWWNs: wwns},
		},
	}

	var mountContext string
	if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) {
		kvh, ok := plugin.host.(volume.KubeletVolumeHost)
		if !ok {
			return volume.ReconstructedVolume{}, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
		}
		hu := kvh.GetHostUtil()
		mountContext, err = hu.GetSELinuxMountContext(mountPath)
		if err != nil {
			return volume.ReconstructedVolume{}, err
		}
	}

	klog.V(5).Infof("ConstructVolumeSpec: TargetWWNs: %v, Lun: %v, WWIDs: %v",
		fcVolume.VolumeSource.FC.TargetWWNs, *fcVolume.VolumeSource.FC.Lun, fcVolume.VolumeSource.FC.WWIDs)
	return volume.ReconstructedVolume{
		Spec:                volume.NewSpecFromVolume(fcVolume),
		SELinuxMountContext: mountContext,
	}, nil
}


// ConstructBlockVolumeSpec creates a new volume.Spec with following steps.
//   - Searches a file whose name is {pod uuid} under volume plugin directory.
//   - If a file is found, then retrieves volumePluginDependentPath from globalMapPathUUID.
//   - Once volumePluginDependentPath is obtained, store volume information to VolumeSource
//
// examples:
//
//	mapPath: pods/{podUid}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName}
//	globalMapPathUUID : plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{pod uuid}
func (plugin *fcPlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mapPath string) (*volume.Spec, error) {
	pluginDir := plugin.host.GetVolumeDevicePluginDir(fcPluginName)
	blkutil := volumepathhandler.NewBlockVolumePathHandler()
	globalMapPathUUID, err := blkutil.FindGlobalMapPathUUIDFromPod(pluginDir, mapPath, podUID)
	if err != nil {
		return nil, err
	}
	klog.V(5).Infof("globalMapPathUUID: %v, err: %v", globalMapPathUUID, err)

	// Retrieve globalPDPath from globalMapPathUUID
	// globalMapPathUUID examples:
	//   wwn+lun: plugins/kubernetes.io/fc/volumeDevices/50060e801049cfd1-lun-0/{pod uuid}
	//   wwid: plugins/kubernetes.io/fc/volumeDevices/3600508b400105e210000900000490000/{pod uuid}
	globalPDPath := filepath.Dir(globalMapPathUUID)
	// Create volume from wwn+lun or wwid
	wwns, lun, wwids, err := parsePDName(globalPDPath)
	if err != nil {
		return nil, fmt.Errorf("failed to retrieve volume plugin information from globalPDPath: %s", err)
	}
	fcPV := createPersistentVolumeFromFCVolumeSource(volumeName,
		v1.FCVolumeSource{TargetWWNs: wwns, Lun: &lun, WWIDs: wwids})
	klog.V(5).Infof("ConstructBlockVolumeSpec: TargetWWNs: %v, Lun: %v, WWIDs: %v",
		fcPV.Spec.PersistentVolumeSource.FC.TargetWWNs,
		*fcPV.Spec.PersistentVolumeSource.FC.Lun,
		fcPV.Spec.PersistentVolumeSource.FC.WWIDs)

	return volume.NewSpecFromPersistentVolume(fcPV, false), nil
}


func (fc *fcDisk) GetPath() string

func (fc *fcDisk) GetPath() string {
	// safe to use PodVolumeDir now: volume teardown occurs before pod is cleaned up
	return fc.plugin.host.GetPodVolumeDir(fc.podUID, utilstrings.EscapeQualifiedName(fcPluginName), fc.volName)
}


func (fc *fcDisk) fcGlobalMapPath(spec *volume.Spec) (string, error) {
	mounter, err := volumeSpecToMounter(spec, fc.plugin.host)
	if err != nil {
		klog.Warningf("failed to get fc mounter: %v", err)
		return "", err
	}
	return fc.manager.MakeGlobalVDPDName(*mounter.fcDisk), nil
}


func (fc *fcDisk) fcPodDeviceMapPath() (string, string) {
	return fc.plugin.host.GetPodVolumeDeviceDir(fc.podUID, utilstrings.EscapeQualifiedName(fcPluginName)), fc.volName
}


func (b *fcDiskMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:       b.readOnly,
		Managed:        !b.readOnly,
		SELinuxRelabel: !b.mountedWithSELinuxContext,
	}
}


func (b *fcDiskMounter) SetUp(mounterArgs volume.MounterArgs) error {
	return b.SetUpAt(b.GetPath(), mounterArgs)
}


func (b *fcDiskMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	// diskSetUp checks mountpoints and prevent repeated calls
	err := diskSetUp(b.manager, *b, dir, b.mounter, mounterArgs.FsGroup, mounterArgs.FSGroupChangePolicy)
	if err != nil {
		klog.Errorf("fc: failed to setup")
	}

	if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) {
		// The volume must have been mounted in MountDevice with -o context.
		b.mountedWithSELinuxContext = mounterArgs.SELinuxLabel != ""
	}
	return err
}


// Unmounts the bind mount, and detaches the disk only if the disk
// resource was the last reference to that disk on the kubelet.
func (c *fcDiskUnmounter) TearDown() error {
	return c.TearDownAt(c.GetPath())
}


func (c *fcDiskUnmounter) TearDownAt(dir string) error {
	return mount.CleanupMountPoint(dir, c.mounter, false)
}


func (c *fcDiskUnmapper) TearDownDevice(mapPath, devicePath string) error {
	err := c.manager.DetachBlockFCDisk(*c, mapPath, devicePath)
	if err != nil {
		return fmt.Errorf("fc: failed to detach disk: %s\nError: %v", mapPath, err)
	}
	klog.V(4).Infof("fc: %s is unmounted, deleting the directory", mapPath)
	if err = os.RemoveAll(mapPath); err != nil {
		return fmt.Errorf("fc: failed to delete the directory: %s\nError: %v", mapPath, err)
	}
	klog.V(4).Infof("fc: successfully detached disk: %s", mapPath)
	return nil
}


func (c *fcDiskUnmapper) UnmapPodDevice() error {
	return nil
}


// GetGlobalMapPath returns global map path and error
// path: plugins/kubernetes.io/{PluginName}/volumeDevices/{WWID}/{podUid}
func (fc *fcDisk) GetGlobalMapPath(spec *volume.Spec) (string, error) {
	return fc.fcGlobalMapPath(spec)
}


// GetPodDeviceMapPath returns pod device map path and volume name
// path: pods/{podUid}/volumeDevices/kubernetes.io~fc
// volumeName: pv0001
func (fc *fcDisk) GetPodDeviceMapPath() (string, string) {
	return fc.fcPodDeviceMapPath()
}


func getVolumeSource(spec *volume.Spec) (*v1.FCVolumeSource, bool, error) {
	// fc volumes used directly in a pod have a ReadOnly flag set by the pod author.
	// fc volumes used as a PersistentVolume gets the ReadOnly flag indirectly through the persistent-claim volume used to mount the PV
	if spec.Volume != nil && spec.Volume.FC != nil {
		return spec.Volume.FC, spec.Volume.FC.ReadOnly, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.FC != nil {
		return spec.PersistentVolume.Spec.FC, spec.ReadOnly, nil
	}

	return nil, false, fmt.Errorf("Spec does not reference a FibreChannel volume type")
}


func createPersistentVolumeFromFCVolumeSource(volumeName string, fc v1.FCVolumeSource) *v1.PersistentVolume {
	block := v1.PersistentVolumeBlock
	return &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: volumeName,
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeSource: v1.PersistentVolumeSource{
				FC: &fc,
			},
			VolumeMode: &block,
		},
	}
}


func getWwnsLunWwids(fc *v1.FCVolumeSource) ([]string, string, []string, error) {
	var lun string
	var wwids []string
	if fc.Lun != nil && len(fc.TargetWWNs) != 0 {
		lun = strconv.Itoa(int(*fc.Lun))
		return fc.TargetWWNs, lun, wwids, nil
	}
	if len(fc.WWIDs) != 0 {
		for _, wwid := range fc.WWIDs {
			wwids = append(wwids, strings.Replace(wwid, " ", "_", -1))
		}
		return fc.TargetWWNs, lun, wwids, nil
	}
	return nil, "", nil, fmt.Errorf("fc: no fc disk information found")
}


func (handler *osIOHandler) ReadDir(dirname string) ([]os.FileInfo, error) {
	return ioutil.ReadDir(dirname)
}


func (handler *osIOHandler) Lstat(name string) (os.FileInfo, error) {
	return os.Lstat(name)
}


func (handler *osIOHandler) EvalSymlinks(path string) (string, error) {
	return filepath.EvalSymlinks(path)
}


func (handler *osIOHandler) WriteFile(filename string, data []byte, perm os.FileMode) error {
	return ioutil.WriteFile(filename, data, perm)
}


// given a wwn and lun, find the device and associated devicemapper parent
func findDisk(wwn, lun string, io ioHandler, deviceUtil volumeutil.DeviceUtil) (string, string) {
	fcPathExp := "^(pci-.*-fc|fc)-0x" + wwn + "-lun-" + lun + "$"
	r := regexp.MustCompile(fcPathExp)
	devPath := byPath
	if dirs, err := io.ReadDir(devPath); err == nil {
		for _, f := range dirs {
			name := f.Name()
			if r.MatchString(name) {
				if disk, err1 := io.EvalSymlinks(devPath + name); err1 == nil {
					dm := deviceUtil.FindMultipathDeviceForDevice(disk)
					klog.Infof("fc: find disk: %v, dm: %v, fc path: %v", disk, dm, name)
					return disk, dm
				}
			}
		}
	}
	return "", ""
}


// given a wwid, find the device and associated devicemapper parent
func findDiskWWIDs(wwid string, io ioHandler, deviceUtil volumeutil.DeviceUtil) (string, string) {
	// Example wwid format:
	//   3600508b400105e210000900000490000
	//   <VENDOR NAME> <IDENTIFIER NUMBER>
	// Example of symlink under by-id:
	//   /dev/by-id/scsi-3600508b400105e210000900000490000
	//   /dev/by-id/scsi-<VENDOR NAME>_<IDENTIFIER NUMBER>
	// The wwid could contain white space and it will be replaced
	// underscore when wwid is exposed under /dev/by-id.

	fcPath := "scsi-" + wwid
	devID := byID
	if dirs, err := io.ReadDir(devID); err == nil {
		for _, f := range dirs {
			name := f.Name()
			if name == fcPath {
				disk, err := io.EvalSymlinks(devID + name)
				if err != nil {
					klog.V(2).Infof("fc: failed to find a corresponding disk from symlink[%s], error %v", devID+name, err)
					return "", ""
				}
				dm := deviceUtil.FindMultipathDeviceForDevice(disk)
				klog.Infof("fc: find disk: %v, dm: %v", disk, dm)
				return disk, dm
			}
		}
	}
	klog.V(2).Infof("fc: failed to find a disk [%s]", devID+fcPath)
	return "", ""
}


// Flushes any outstanding I/O to the device
func flushDevice(deviceName string, exec utilexec.Interface) {
	out, err := exec.Command("blockdev", "--flushbufs", deviceName).CombinedOutput()
	if err != nil {
		// Ignore the error and continue deleting the device. There is will be no retry on error.
		klog.Warningf("Failed to flush device %s: %s\n%s", deviceName, err, string(out))
	}
	klog.V(4).Infof("Flushed device %s", deviceName)
}


// Removes a scsi device based upon /dev/sdX name
func removeFromScsiSubsystem(deviceName string, io ioHandler) {
	fileName := "/sys/block/" + deviceName + "/device/delete"
	klog.V(4).Infof("fc: remove device from scsi-subsystem: path: %s", fileName)
	data := []byte("1")
	io.WriteFile(fileName, data, 0666)
}


func scsiHostRescan(io ioHandler)

// rescan scsi bus
func scsiHostRescan(io ioHandler) {
	scsiPath := "/sys/class/scsi_host/"
	if dirs, err := io.ReadDir(scsiPath); err == nil {
		for _, f := range dirs {
			name := scsiPath + f.Name() + "/scan"
			data := []byte("- - -")
			io.WriteFile(name, data, 0666)
		}
	}
}


// make a directory like /var/lib/kubelet/plugins/kubernetes.io/fc/target1-target2-lun-0
func makePDNameInternal(host volume.VolumeHost, wwns []string, lun string, wwids []string) string {
	if len(wwns) != 0 {
		w := strings.Join(wwns, "-")
		return filepath.Join(host.GetPluginDir(fcPluginName), w+"-lun-"+lun)
	}
	return filepath.Join(host.GetPluginDir(fcPluginName), strings.Join(wwids, "-"))
}


// make a directory like /var/lib/kubelet/plugins/kubernetes.io/fc/volumeDevices/target-lun-0
func makeVDPDNameInternal(host volume.VolumeHost, wwns []string, lun string, wwids []string) string {
	if len(wwns) != 0 {
		w := strings.Join(wwns, "-")
		return filepath.Join(host.GetVolumeDevicePluginDir(fcPluginName), w+"-lun-"+lun)
	}
	return filepath.Join(host.GetVolumeDevicePluginDir(fcPluginName), strings.Join(wwids, "-"))
}


func parsePDName(path string) (wwns []string, lun int32, wwids []string, err error) {
	// parse directory name created by makePDNameInternal or makeVDPDNameInternal
	dirname := filepath.Base(path)
	components := strings.Split(dirname, "-")
	l := len(components)
	if l == 1 {
		// No '-', it must be single WWID
		return nil, 0, components, nil
	}
	if components[l-2] == "lun" {
		// it has -lun-, it's list of WWNs + lun number as the last component
		if l == 2 {
			return nil, 0, nil, fmt.Errorf("no wwn in: %s", dirname)
		}
		lun, err := strconv.Atoi(components[l-1])
		if err != nil {
			return nil, 0, nil, err
		}

		return components[:l-2], int32(lun), nil, nil
	}
	// no -lun-, it's just list of WWIDs
	return nil, 0, components, nil
}


func (util *fcUtil) MakeGlobalPDName(fc fcDisk) string {
	return makePDNameInternal(fc.plugin.host, fc.wwns, fc.lun, fc.wwids)
}


// Global volume device plugin dir
func (util *fcUtil) MakeGlobalVDPDName(fc fcDisk) string {
	return makeVDPDNameInternal(fc.plugin.host, fc.wwns, fc.lun, fc.wwids)
}


func searchDisk(b fcDiskMounter) (string, error) {
	var diskIDs []string
	var disk string
	var dm string
	io := b.io
	wwids := b.wwids
	wwns := b.wwns
	lun := b.lun

	if len(wwns) != 0 {
		diskIDs = wwns
	} else {
		diskIDs = wwids
	}

	rescanned := false
	// two-phase search:
	// first phase, search existing device path, if a multipath dm is found, exit loop
	// otherwise, in second phase, rescan scsi bus and search again, return with any findings
	for true {
		for _, diskID := range diskIDs {
			if len(wwns) != 0 {
				disk, dm = findDisk(diskID, lun, io, b.deviceUtil)
			} else {
				disk, dm = findDiskWWIDs(diskID, io, b.deviceUtil)
			}
			// if multipath device is found, break
			if dm != "" {
				break
			}
		}
		// if a dm is found, exit loop
		if rescanned || dm != "" {
			break
		}
		// rescan and search again
		// rescan scsi bus
		scsiHostRescan(io)
		rescanned = true
	}
	// if no disk matches input wwn and lun, exit
	if disk == "" && dm == "" {
		return "", fmt.Errorf("no fc disk found")
	}

	// if multipath devicemapper device is found, use it; otherwise use raw disk
	if dm != "" {
		return dm, nil
	}
	return disk, nil
}


func (util *fcUtil) AttachDisk(b fcDiskMounter) (string, error) {
	devicePath, err := searchDisk(b)
	if err != nil {
		return "", err
	}

	exists, err := mount.PathExists(devicePath)
	if exists && err == nil {
		return devicePath, nil
	}
	if exists == false {
		return "", fmt.Errorf("device %s does not exist", devicePath)
	} else {
		return "", err
	}
}


// DetachDisk removes scsi device file such as /dev/sdX from the node.
func (util *fcUtil) DetachDisk(c fcDiskUnmounter, devicePath string) error {
	var devices []string
	// devicePath might be like /dev/mapper/mpathX. Find destination.
	dstPath, err := c.io.EvalSymlinks(devicePath)
	if err != nil {
		return err
	}
	// Find slave
	if strings.HasPrefix(dstPath, "/dev/dm-") {
		devices = c.deviceUtil.FindSlaveDevicesOnMultipath(dstPath)
		if err := util.deleteMultipathDevice(c.exec, dstPath); err != nil {
			return err
		}
	} else {
		// Add single devicepath to devices
		devices = append(devices, dstPath)
	}
	klog.V(4).Infof("fc: DetachDisk devicePath: %v, dstPath: %v, devices: %v", devicePath, dstPath, devices)
	var lastErr error
	for _, device := range devices {
		err := util.detachFCDisk(c.io, c.exec, device)
		if err != nil {
			klog.Errorf("fc: detachFCDisk failed. device: %v err: %v", device, err)
			lastErr = fmt.Errorf("fc: detachFCDisk failed. device: %v err: %v", device, err)
		}
	}
	if lastErr != nil {
		klog.Errorf("fc: last error occurred during detach disk:\n%v", lastErr)
		return lastErr
	}
	return nil
}


// detachFCDisk removes scsi device file such as /dev/sdX from the node.
func (util *fcUtil) detachFCDisk(io ioHandler, exec utilexec.Interface, devicePath string) error {
	// Remove scsi device from the node.
	if !strings.HasPrefix(devicePath, "/dev/") {
		return fmt.Errorf("fc detach disk: invalid device name: %s", devicePath)
	}
	flushDevice(devicePath, exec)
	arr := strings.Split(devicePath, "/")
	dev := arr[len(arr)-1]
	removeFromScsiSubsystem(dev, io)
	return nil
}


// DetachBlockFCDisk detaches a volume from kubelet node, removes scsi device file
// such as /dev/sdX from the node, and then removes loopback for the scsi device.
func (util *fcUtil) DetachBlockFCDisk(c fcDiskUnmapper, mapPath, devicePath string) error {
	// Check if devicePath is valid
	if len(devicePath) != 0 {
		if pathExists, pathErr := checkPathExists(devicePath); !pathExists || pathErr != nil {
			return pathErr
		}
	} else {
		// TODO: FC plugin can't obtain the devicePath from kubelet because devicePath
		// in volume object isn't updated when volume is attached to kubelet node.
		klog.Infof("fc: devicePath is empty. Try to retrieve FC configuration from global map path: %v", mapPath)
	}

	// Check if global map path is valid
	// global map path examples:
	//   wwn+lun: plugins/kubernetes.io/fc/volumeDevices/50060e801049cfd1-lun-0/
	//   wwid: plugins/kubernetes.io/fc/volumeDevices/3600508b400105e210000900000490000/
	if pathExists, pathErr := checkPathExists(mapPath); !pathExists || pathErr != nil {
		return pathErr
	}

	// Retrieve volume plugin dependent path like '50060e801049cfd1-lun-0' from global map path
	arr := strings.Split(mapPath, "/")
	if len(arr) < 1 {
		return fmt.Errorf("failed to retrieve volume plugin information from global map path: %v", mapPath)
	}
	volumeInfo := arr[len(arr)-1]

	// Search symbolic link which matches volumeInfo under /dev/disk/by-path or /dev/disk/by-id
	// then find destination device path from the link
	searchPath := byID
	if strings.Contains(volumeInfo, "-lun-") {
		searchPath = byPath
	}
	fis, err := ioutil.ReadDir(searchPath)
	if err != nil {
		return err
	}
	for _, fi := range fis {
		if strings.Contains(fi.Name(), volumeInfo) {
			devicePath = filepath.Join(searchPath, fi.Name())
			klog.V(5).Infof("fc: updated devicePath: %s", devicePath)
			break
		}
	}
	if len(devicePath) == 0 {
		return fmt.Errorf("fc: failed to find corresponding device from searchPath: %v", searchPath)
	}
	dstPath, err := c.io.EvalSymlinks(devicePath)
	if err != nil {
		return err
	}
	klog.V(4).Infof("fc: find destination device path from symlink: %v", dstPath)

	var devices []string
	dm := c.deviceUtil.FindMultipathDeviceForDevice(dstPath)
	if len(dm) != 0 {
		dstPath = dm
	}

	// Detach volume from kubelet node
	if len(dm) != 0 {
		// Find all devices which are managed by multipath
		devices = c.deviceUtil.FindSlaveDevicesOnMultipath(dm)
		if err := util.deleteMultipathDevice(c.exec, dm); err != nil {
			return err
		}
	} else {
		// Add single device path to devices
		devices = append(devices, dstPath)
	}
	var lastErr error
	for _, device := range devices {
		err = util.detachFCDisk(c.io, c.exec, device)
		if err != nil {
			klog.Errorf("fc: detachFCDisk failed. device: %v err: %v", device, err)
			lastErr = fmt.Errorf("fc: detachFCDisk failed. device: %v err: %v", device, err)
		}
	}
	if lastErr != nil {
		klog.Errorf("fc: last error occurred during detach disk:\n%v", lastErr)
		return lastErr
	}
	return nil
}


func (util *fcUtil) deleteMultipathDevice(exec utilexec.Interface, dmDevice string) error {
	out, err := exec.Command("multipath", "-f", dmDevice).CombinedOutput()
	if err != nil {
		return fmt.Errorf("failed to flush multipath device %s: %s\n%s", dmDevice, err, string(out))
	}
	klog.V(4).Infof("Flushed multipath device: %s", dmDevice)
	return nil
}


func checkPathExists(path string) (bool, error) {
	if pathExists, pathErr := mount.PathExists(path); pathErr != nil {
		return pathExists, fmt.Errorf("error checking if path exists: %w", pathErr)
	} else if !pathExists {
		klog.Warningf("Warning: Unmap skipped because path does not exist: %v", path)
		return pathExists, nil
	}
	return true, nil
}


// Attach is part of the volume.Attacher interface
func (a *attacherDefaults) Attach(spec *volume.Spec, hostName types.NodeName) (string, error) {
	klog.Warning(logPrefix(a.plugin.flexVolumePlugin), "using default Attach for volume ", spec.Name(), ", host ", hostName)
	return "", nil
}


// WaitForAttach is part of the volume.Attacher interface
func (a *attacherDefaults) WaitForAttach(spec *volume.Spec, devicePath string, timeout time.Duration) (string, error) {
	klog.Warning(logPrefix(a.plugin.flexVolumePlugin), "using default WaitForAttach for volume ", spec.Name(), ", device ", devicePath)
	return devicePath, nil
}


// GetDeviceMountPath is part of the volume.Attacher interface
func (a *attacherDefaults) GetDeviceMountPath(spec *volume.Spec, mountsDir string) (string, error) {
	return a.plugin.getDeviceMountPath(spec)
}


// MountDevice is part of the volume.Attacher interface
func (a *attacherDefaults) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string, mounter mount.Interface) error {
	klog.Warning(logPrefix(a.plugin.flexVolumePlugin), "using default MountDevice for volume ", spec.Name(), ", device ", devicePath, ", deviceMountPath ", deviceMountPath)

	volSourceFSType, err := getFSType(spec)
	if err != nil {
		return err
	}

	readOnly, err := getReadOnly(spec)
	if err != nil {
		return err
	}

	options := make([]string, 0)

	if readOnly {
		options = append(options, "ro")
	} else {
		options = append(options, "rw")
	}

	diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Exec: a.plugin.host.GetExec(a.plugin.GetPluginName())}

	return diskMounter.FormatAndMount(devicePath, deviceMountPath, volSourceFSType, options)
}


// Attach is part of the volume.Attacher interface
func (a *flexVolumeAttacher) Attach(spec *volume.Spec, hostName types.NodeName) (string, error) {

	call := a.plugin.NewDriverCall(attachCmd)
	call.AppendSpec(spec, a.plugin.host, nil)
	call.Append(string(hostName))

	status, err := call.Run()
	if isCmdNotSupportedErr(err) {
		return (*attacherDefaults)(a).Attach(spec, hostName)
	} else if err != nil {
		return "", err
	}
	return status.DevicePath, err
}


// WaitForAttach is part of the volume.Attacher interface
func (a *flexVolumeAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) {
	call := a.plugin.NewDriverCallWithTimeout(waitForAttachCmd, timeout)
	call.Append(devicePath)
	call.AppendSpec(spec, a.plugin.host, nil)

	status, err := call.Run()
	if isCmdNotSupportedErr(err) {
		return (*attacherDefaults)(a).WaitForAttach(spec, devicePath, timeout)
	} else if err != nil {
		return "", err
	}
	return status.DevicePath, nil
}


// GetDeviceMountPath is part of the volume.Attacher interface
func (a *flexVolumeAttacher) GetDeviceMountPath(spec *volume.Spec) (string, error) {
	return a.plugin.getDeviceMountPath(spec)
}


// MountDevice is part of the volume.Attacher interface
func (a *flexVolumeAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string, _ volume.DeviceMounterArgs) error {
	// Mount only once.
	alreadyMounted, err := prepareForMount(a.plugin.host.GetMounter(a.plugin.GetPluginName()), deviceMountPath)
	if err != nil {
		return err
	}
	if alreadyMounted {
		return nil
	}

	call := a.plugin.NewDriverCall(mountDeviceCmd)
	call.Append(deviceMountPath)
	call.Append(devicePath)
	call.AppendSpec(spec, a.plugin.host, nil)

	_, err = call.Run()
	if isCmdNotSupportedErr(err) {
		// Devicepath is empty if the plugin does not support attach calls. Ignore mountDevice calls if the
		// plugin does not implement attach interface.
		if devicePath != "" {
			return (*attacherDefaults)(a).MountDevice(spec, devicePath, deviceMountPath, a.plugin.host.GetMounter(a.plugin.GetPluginName()))
		}
		return nil
	}
	return err
}


func (a *flexVolumeAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.NodeName) (map[*volume.Spec]bool, error) {
	volumesAttachedCheck := make(map[*volume.Spec]bool)
	for _, spec := range specs {
		volumesAttachedCheck[spec] = true

		call := a.plugin.NewDriverCall(isAttached)
		call.AppendSpec(spec, a.plugin.host, nil)
		call.Append(string(nodeName))

		status, err := call.Run()
		if isCmdNotSupportedErr(err) {
			return nil, nil
		} else if err == nil {
			if !status.Attached {
				volumesAttachedCheck[spec] = false
				klog.V(2).Infof("VolumesAreAttached: check volume (%q) is no longer attached", spec.Name())
			}
		} else {
			return nil, err
		}
	}
	return volumesAttachedCheck, nil
}


// Detach is part of the volume.Detacher interface.
func (d *detacherDefaults) Detach(volumeName string, hostName types.NodeName) error {
	klog.Warning(logPrefix(d.plugin.flexVolumePlugin), "using default Detach for volume ", volumeName, ", host ", hostName)
	return nil
}


// WaitForDetach is part of the volume.Detacher interface.
func (d *detacherDefaults) WaitForDetach(devicePath string, timeout time.Duration) error {
	klog.Warning(logPrefix(d.plugin.flexVolumePlugin), "using default WaitForDetach for device ", devicePath)
	return nil
}


// UnmountDevice is part of the volume.Detacher interface.
func (d *detacherDefaults) UnmountDevice(deviceMountPath string) error {
	klog.Warning(logPrefix(d.plugin.flexVolumePlugin), "using default UnmountDevice for device mount path ", deviceMountPath)
	return mount.CleanupMountPoint(deviceMountPath, d.plugin.host.GetMounter(d.plugin.GetPluginName()), false)
}


// Detach is part of the volume.Detacher interface.
func (d *flexVolumeDetacher) Detach(volumeName string, hostName types.NodeName) error {

	call := d.plugin.NewDriverCall(detachCmd)
	call.Append(volumeName)
	call.Append(string(hostName))

	_, err := call.Run()
	if isCmdNotSupportedErr(err) {
		return (*detacherDefaults)(d).Detach(volumeName, hostName)
	}
	return err
}


// UnmountDevice is part of the volume.Detacher interface.
func (d *flexVolumeDetacher) UnmountDevice(deviceMountPath string) error {

	pathExists, pathErr := mount.PathExists(deviceMountPath)
	if !pathExists {
		klog.Warningf("Warning: Unmount skipped because path does not exist: %v", deviceMountPath)
		return nil
	}
	if pathErr != nil && !mount.IsCorruptedMnt(pathErr) {
		return fmt.Errorf("error checking path: %w", pathErr)
	}

	notmnt, err := isNotMounted(d.plugin.host.GetMounter(d.plugin.GetPluginName()), deviceMountPath)
	if err != nil {
		if mount.IsCorruptedMnt(err) {
			notmnt = false // Corrupted error is assumed to be mounted.
		} else {
			return err
		}
	}

	if notmnt {
		klog.Warningf("Warning: Path: %v already unmounted", deviceMountPath)
	} else {
		call := d.plugin.NewDriverCall(unmountDeviceCmd)
		call.Append(deviceMountPath)

		_, err := call.Run()
		if isCmdNotSupportedErr(err) {
			err = (*detacherDefaults)(d).UnmountDevice(deviceMountPath)
		}
		if err != nil {
			return err
		}
	}

	// Flexvolume driver may remove the directory. Ignore if it does.
	if pathExists, pathErr := mount.PathExists(deviceMountPath); pathErr != nil {
		return fmt.Errorf("error checking if path exists: %w", pathErr)
	} else if !pathExists {
		return nil
	}
	return os.Remove(deviceMountPath)
}


func (plugin *flexVolumePlugin) NewDriverCall(command string) *DriverCall {
	return plugin.NewDriverCallWithTimeout(command, 0)
}


func (plugin *flexVolumePlugin) NewDriverCallWithTimeout(command string, timeout time.Duration) *DriverCall {
	return &DriverCall{
		Command: command,
		Timeout: timeout,
		plugin:  plugin,
		args:    []string{command},
	}
}


// Append appends arg into driver call argument list
func (dc *DriverCall) Append(arg string) {
	dc.args = append(dc.args, arg)
}


// AppendSpec appends volume spec to driver call argument list
func (dc *DriverCall) AppendSpec(spec *volume.Spec, host volume.VolumeHost, extraOptions map[string]string) error {
	optionsForDriver, err := NewOptionsForDriver(spec, host, extraOptions)
	if err != nil {
		return err
	}

	jsonBytes, err := json.Marshal(optionsForDriver)
	if err != nil {
		return fmt.Errorf("failed to marshal spec, error: %s", err.Error())
	}

	dc.Append(string(jsonBytes))
	return nil
}


// Run executes the driver call
func (dc *DriverCall) Run() (*DriverStatus, error) {
	if dc.plugin.isUnsupported(dc.Command) {
		return nil, errors.New(StatusNotSupported)
	}
	execPath := dc.plugin.getExecutable()

	cmd := dc.plugin.runner.Command(execPath, dc.args...)

	timeout := false
	if dc.Timeout > 0 {
		timer := time.AfterFunc(dc.Timeout, func() {
			timeout = true
			cmd.Stop()
		})
		defer timer.Stop()
	}

	output, execErr := cmd.CombinedOutput()
	if execErr != nil {
		if timeout {
			return nil, errTimeout
		}
		_, err := handleCmdResponse(dc.Command, output)
		if err == nil {
			klog.Errorf("FlexVolume: driver bug: %s: exec error (%s) but no error in response.", execPath, execErr)
			return nil, execErr
		}
		if isCmdNotSupportedErr(err) {
			dc.plugin.unsupported(dc.Command)
		} else {
			klog.Warningf("FlexVolume: driver call failed: executable: %s, args: %s, error: %s, output: %q", execPath, dc.args, execErr.Error(), output)
		}
		return nil, err
	}

	status, err := handleCmdResponse(dc.Command, output)
	if err != nil {
		if isCmdNotSupportedErr(err) {
			dc.plugin.unsupported(dc.Command)
		}
		return nil, err
	}

	return status, nil
}


// NewOptionsForDriver create driver options given volume spec
func NewOptionsForDriver(spec *volume.Spec, host volume.VolumeHost, extraOptions map[string]string) (OptionsForDriver, error) {

	volSourceFSType, err := getFSType(spec)
	if err != nil {
		return nil, err
	}

	readOnly, err := getReadOnly(spec)
	if err != nil {
		return nil, err
	}

	volSourceOptions, err := getOptions(spec)
	if err != nil {
		return nil, err
	}

	options := map[string]string{}

	options[optionFSType] = volSourceFSType

	if readOnly {
		options[optionReadWrite] = "ro"
	} else {
		options[optionReadWrite] = "rw"
	}

	options[optionPVorVolumeName] = spec.Name()

	for key, value := range extraOptions {
		options[key] = value
	}

	for key, value := range volSourceOptions {
		options[key] = value
	}

	return OptionsForDriver(options), nil
}


func defaultCapabilities() *DriverCapabilities {
	return &DriverCapabilities{
		Attach:           true,
		SELinuxRelabel:   true,
		SupportsMetrics:  false,
		FSGroup:          true,
		RequiresFSResize: true,
	}
}


// isCmdNotSupportedErr checks if the error corresponds to command not supported by
// driver.
func isCmdNotSupportedErr(err error) bool {
	return err != nil && err.Error() == StatusNotSupported
}


// handleCmdResponse processes the command output and returns the appropriate
// error code or message.
func handleCmdResponse(cmd string, output []byte) (*DriverStatus, error) {
	status := DriverStatus{
		Capabilities: defaultCapabilities(),
	}
	if err := json.Unmarshal(output, &status); err != nil {
		klog.Errorf("Failed to unmarshal output for command: %s, output: %q, error: %s", cmd, string(output), err.Error())
		return nil, err
	} else if status.Status == StatusNotSupported {
		klog.V(5).Infof("%s command is not supported by the driver", cmd)
		return nil, errors.New(status.Status)
	} else if status.Status != StatusSuccess {
		errMsg := fmt.Sprintf("%s command failed, status: %s, reason: %s", cmd, status.Status, status.Message)
		klog.Errorf(errMsg)
		return nil, fmt.Errorf("%s", errMsg)
	}

	return &status, nil
}


func newExpanderDefaults(plugin *flexVolumePlugin) *expanderDefaults {
	return &expanderDefaults{plugin}
}


func (e *expanderDefaults) ExpandVolumeDevice(spec *volume.Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) {
	klog.Warning(logPrefix(e.plugin), "using default expand for volume ", spec.Name(), ", to size ", newSize, " from ", oldSize)
	return newSize, nil
}


// the defaults for NodeExpand return a generic resize indicator that will trigger the operation executor to go ahead with
// generic filesystem resize
func (e *expanderDefaults) NodeExpand(rsOpt volume.NodeResizeOptions) (bool, error) {
	klog.Warning(logPrefix(e.plugin), "using default filesystem resize for volume ", rsOpt.VolumeSpec.Name(), ", at ", rsOpt.DevicePath)
	fsVolume, err := util.CheckVolumeModeFilesystem(rsOpt.VolumeSpec)
	if err != nil {
		return false, fmt.Errorf("error checking VolumeMode: %v", err)
	}
	// if volume is not a fs file system, there is nothing for us to do here.
	if !fsVolume {
		return true, nil
	}

	_, err = util.GenericResizeFS(e.plugin.host, e.plugin.GetPluginName(), rsOpt.DevicePath, rsOpt.DeviceMountPath)
	if err != nil {
		return false, err
	}
	return true, nil
}


func (plugin *flexVolumePlugin) ExpandVolumeDevice(spec *volume.Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) {
	call := plugin.NewDriverCall(expandVolumeCmd)
	call.AppendSpec(spec, plugin.host, nil)

	devicePath, err := plugin.getDeviceMountPath(spec)
	if err != nil {
		return newSize, err
	}
	call.Append(devicePath)
	call.Append(strconv.FormatInt(newSize.Value(), 10))
	call.Append(strconv.FormatInt(oldSize.Value(), 10))

	_, err = call.Run()
	if isCmdNotSupportedErr(err) {
		return newExpanderDefaults(plugin).ExpandVolumeDevice(spec, newSize, oldSize)
	}
	return newSize, err
}


func (plugin *flexVolumePlugin) NodeExpand(rsOpt volume.NodeResizeOptions) (bool, error) {
	// This method is called after we spec.PersistentVolume.Spec.Capacity
	// has been updated to the new size. The underlying driver thus sees
	// the _new_ (requested) size and can find out the _current_ size from
	// its underlying storage implementation

	if rsOpt.VolumeSpec.PersistentVolume == nil {
		return false, fmt.Errorf("PersistentVolume not found for spec: %s", rsOpt.VolumeSpec.Name())
	}

	call := plugin.NewDriverCall(expandFSCmd)
	call.AppendSpec(rsOpt.VolumeSpec, plugin.host, nil)
	call.Append(rsOpt.DevicePath)
	call.Append(rsOpt.DeviceMountPath)
	call.Append(strconv.FormatInt(rsOpt.NewSize.Value(), 10))
	call.Append(strconv.FormatInt(rsOpt.OldSize.Value(), 10))

	_, err := call.Run()
	if isCmdNotSupportedErr(err) {
		return newExpanderDefaults(plugin).NodeExpand(rsOpt)
	}
	if err != nil {
		return false, err
	}
	return true, nil
}


// SetUpAt is part of the volume.Mounter interface.
// This implementation relies on the attacher's device mount path and does a bind mount to dir.
func (f *mounterDefaults) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	klog.Warning(logPrefix(f.plugin), "using default SetUpAt to ", dir)

	src, err := f.plugin.getDeviceMountPath(f.spec)
	if err != nil {
		return err
	}

	if err := doMount(f.mounter, src, dir, "auto", []string{"bind"}); err != nil {
		return err
	}

	return nil
}


// Returns the default volume attributes.
func (f *mounterDefaults) GetAttributes() volume.Attributes {
	klog.V(5).Info(logPrefix(f.plugin), "using default GetAttributes")
	return volume.Attributes{
		ReadOnly:       f.readOnly,
		Managed:        !f.readOnly,
		SELinuxRelabel: f.flexVolume.plugin.capabilities.SELinuxRelabel,
	}
}


// SetUp creates new directory.
func (f *flexVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error {
	return f.SetUpAt(f.GetPath(), mounterArgs)
}


// SetUpAt creates new directory.
func (f *flexVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	// Mount only once.
	alreadyMounted, err := prepareForMount(f.mounter, dir)
	if err != nil {
		return err
	}
	if alreadyMounted {
		return nil
	}

	call := f.plugin.NewDriverCall(mountCmd)

	// Interface parameters
	call.Append(dir)

	extraOptions := make(map[string]string)

	// pod metadata
	extraOptions[optionKeyPodName] = f.podName
	extraOptions[optionKeyPodNamespace] = f.podNamespace
	extraOptions[optionKeyPodUID] = string(f.podUID)
	// service account metadata
	extraOptions[optionKeyServiceAccountName] = f.podServiceAccountName

	// Extract secret and pass it as options.
	if err := addSecretsToOptions(extraOptions, f.spec, f.podNamespace, f.driverName, f.plugin.host); err != nil {
		os.Remove(dir)
		return err
	}

	// Implicit parameters
	if mounterArgs.FsGroup != nil {
		extraOptions[optionFSGroup] = strconv.FormatInt(int64(*mounterArgs.FsGroup), 10)
	}

	call.AppendSpec(f.spec, f.plugin.host, extraOptions)

	_, err = call.Run()
	if isCmdNotSupportedErr(err) {
		err = (*mounterDefaults)(f).SetUpAt(dir, mounterArgs)
	}

	if err != nil {
		os.Remove(dir)
		return err
	}

	if !f.readOnly {
		if f.plugin.capabilities.FSGroup {
			// fullPluginName helps to distinguish different driver from flex volume plugin
			volume.SetVolumeOwnership(f, dir, mounterArgs.FsGroup, mounterArgs.FSGroupChangePolicy, util.FSGroupCompleteHook(f.plugin, f.spec))
		}
	}

	return nil
}


// GetAttributes get the flex volume attributes. The attributes will be queried
// using plugin callout after we finalize the callout syntax.
func (f *flexVolumeMounter) GetAttributes() volume.Attributes {
	return (*mounterDefaults)(f).GetAttributes()
}


func logPrefix(plugin *flexVolumePlugin) string {
	return "flexVolume driver " + plugin.driverName + ": "
}


func (plugin *pluginDefaults) GetVolumeName(spec *volume.Spec) (string, error) {
	klog.V(4).Info(logPrefix((*flexVolumePlugin)(plugin)), "using default GetVolumeName for volume ", spec.Name())
	return spec.Name(), nil
}


func (pluginFactory) NewFlexVolumePlugin(pluginDir, name string, runner exec.Interface) (volume.VolumePlugin, error) {
	execPath := filepath.Join(pluginDir, name)

	driverName := utilstrings.UnescapeQualifiedName(name)

	flexPlugin := &flexVolumePlugin{
		driverName:          driverName,
		execPath:            execPath,
		runner:              runner,
		unsupportedCommands: []string{},
	}

	// Initialize the plugin and probe the capabilities
	call := flexPlugin.NewDriverCall(initCmd)
	ds, err := call.Run()
	if err != nil {
		return nil, err
	}
	flexPlugin.capabilities = *ds.Capabilities

	if flexPlugin.capabilities.Attach {
		// Plugin supports attach/detach, so return flexVolumeAttachablePlugin
		return &flexVolumeAttachablePlugin{flexVolumePlugin: flexPlugin}, nil
	}
	return flexPlugin, nil
}


// Init is part of the volume.VolumePlugin interface.
func (plugin *flexVolumePlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	// Hardwired 'success' as any errors from calling init() will be caught by NewFlexVolumePlugin()
	return nil
}


func (plugin *flexVolumePlugin) getExecutable() string {
	parts := strings.Split(plugin.driverName, "/")
	execName := parts[len(parts)-1]
	execPath := filepath.Join(plugin.execPath, execName)
	if runtime.GOOS == "windows" {
		execPath = util.GetWindowsPath(execPath)
	}
	return execPath
}


// Name is part of the volume.VolumePlugin interface.
func (plugin *flexVolumePlugin) GetPluginName() string {
	return plugin.driverName
}


// GetVolumeName is part of the volume.VolumePlugin interface.
func (plugin *flexVolumePlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	call := plugin.NewDriverCall(getVolumeNameCmd)
	call.AppendSpec(spec, plugin.host, nil)

	_, err := call.Run()
	if isCmdNotSupportedErr(err) {
		return (*pluginDefaults)(plugin).GetVolumeName(spec)
	} else if err != nil {
		return "", err
	}

	name, err := (*pluginDefaults)(plugin).GetVolumeName(spec)
	if err != nil {
		return "", err
	}

	klog.V(4).Info(logPrefix(plugin), "GetVolumeName is not supported yet. Defaulting to PV or volume name: ", name)

	return name, nil
}


// CanSupport is part of the volume.VolumePlugin interface.
func (plugin *flexVolumePlugin) CanSupport(spec *volume.Spec) bool {
	sourceDriver, err := getDriver(spec)
	if err != nil {
		return false
	}
	return sourceDriver == plugin.driverName
}


// RequiresRemount is part of the volume.VolumePlugin interface.
func (plugin *flexVolumePlugin) RequiresRemount(spec *volume.Spec) bool {
	return false
}


// GetAccessModes gets the allowed access modes for this plugin.
func (plugin *flexVolumePlugin) GetAccessModes() []api.PersistentVolumeAccessMode {
	return []api.PersistentVolumeAccessMode{
		api.ReadWriteOnce,
		api.ReadOnlyMany,
	}
}


// NewMounter is part of the volume.VolumePlugin interface.
func (plugin *flexVolumePlugin) NewMounter(spec *volume.Spec, pod *api.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
	return plugin.newMounterInternal(spec, pod, plugin.host.GetMounter(plugin.GetPluginName()), plugin.runner)
}


// newMounterInternal is the internal mounter routine to build the volume.
func (plugin *flexVolumePlugin) newMounterInternal(spec *volume.Spec, pod *api.Pod, mounter mount.Interface, runner exec.Interface) (volume.Mounter, error) {
	sourceDriver, err := getDriver(spec)
	if err != nil {
		return nil, err
	}

	readOnly, err := getReadOnly(spec)
	if err != nil {
		return nil, err
	}

	var metricsProvider volume.MetricsProvider
	if plugin.capabilities.SupportsMetrics {
		metricsProvider = volume.NewMetricsStatFS(plugin.host.GetPodVolumeDir(
			pod.UID, utilstrings.EscapeQualifiedName(sourceDriver), spec.Name()))
	} else {
		metricsProvider = &volume.MetricsNil{}
	}

	return &flexVolumeMounter{
		flexVolume: &flexVolume{
			driverName:            sourceDriver,
			execPath:              plugin.getExecutable(),
			mounter:               mounter,
			plugin:                plugin,
			podName:               pod.Name,
			podUID:                pod.UID,
			podNamespace:          pod.Namespace,
			podServiceAccountName: pod.Spec.ServiceAccountName,
			volName:               spec.Name(),
			MetricsProvider:       metricsProvider,
		},
		runner:   runner,
		spec:     spec,
		readOnly: readOnly,
	}, nil
}


// NewUnmounter is part of the volume.VolumePlugin interface.
func (plugin *flexVolumePlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter(plugin.GetPluginName()), plugin.runner)
}


// newUnmounterInternal is the internal unmounter routine to clean the volume.
func (plugin *flexVolumePlugin) newUnmounterInternal(volName string, podUID types.UID, mounter mount.Interface, runner exec.Interface) (volume.Unmounter, error) {
	var metricsProvider volume.MetricsProvider
	if plugin.capabilities.SupportsMetrics {
		metricsProvider = volume.NewMetricsStatFS(plugin.host.GetPodVolumeDir(
			podUID, utilstrings.EscapeQualifiedName(plugin.driverName), volName))
	} else {
		metricsProvider = &volume.MetricsNil{}
	}

	return &flexVolumeUnmounter{
		flexVolume: &flexVolume{
			driverName:      plugin.driverName,
			execPath:        plugin.getExecutable(),
			mounter:         mounter,
			plugin:          plugin,
			podUID:          podUID,
			volName:         volName,
			MetricsProvider: metricsProvider,
		},
		runner: runner,
	}, nil
}


// NewAttacher is part of the volume.AttachableVolumePlugin interface.
func (plugin *flexVolumeAttachablePlugin) NewAttacher() (volume.Attacher, error) {
	return &flexVolumeAttacher{plugin}, nil
}


func (plugin *flexVolumeAttachablePlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
	return plugin.NewAttacher()
}


// NewDetacher is part of the volume.AttachableVolumePlugin interface.
func (plugin *flexVolumeAttachablePlugin) NewDetacher() (volume.Detacher, error) {
	return &flexVolumeDetacher{plugin}, nil
}


func (plugin *flexVolumeAttachablePlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
	return plugin.NewDetacher()
}


func (plugin *flexVolumeAttachablePlugin) CanAttach(spec *volume.Spec) (bool, error) {
	return true, nil
}


func (plugin *flexVolumeAttachablePlugin) CanDeviceMount(spec *volume.Spec) (bool, error) {
	return true, nil
}


// ConstructVolumeSpec is part of the volume.AttachableVolumePlugin interface.
func (plugin *flexVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
	flexVolume := &api.Volume{
		Name: volumeName,
		VolumeSource: api.VolumeSource{
			FlexVolume: &api.FlexVolumeSource{
				Driver: plugin.driverName,
			},
		},
	}
	return volume.ReconstructedVolume{
		Spec: volume.NewSpecFromVolume(flexVolume),
	}, nil
}


func (plugin *flexVolumePlugin) SupportsMountOption() bool {
	return false
}


// Mark the given commands as unsupported.
func (plugin *flexVolumePlugin) unsupported(commands ...string) {
	plugin.Lock()
	defer plugin.Unlock()
	plugin.unsupportedCommands = append(plugin.unsupportedCommands, commands...)
}


func (plugin *flexVolumePlugin) SupportsBulkVolumeVerification() bool {
	return false
}


func (plugin *flexVolumePlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return false, nil
}


// Returns true iff the given command is known to be unsupported.
func (plugin *flexVolumePlugin) isUnsupported(command string) bool {
	plugin.Lock()
	defer plugin.Unlock()
	for _, unsupportedCommand := range plugin.unsupportedCommands {
		if command == unsupportedCommand {
			return true
		}
	}
	return false
}


func (plugin *flexVolumePlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
	mounter := plugin.host.GetMounter(plugin.GetPluginName())
	return mounter.GetMountRefs(deviceMountPath)
}