getGlobalLocalPath

func (plugin *localVolumePlugin) getGlobalLocalPath(spec *volume.Spec) (string, error) {
	if spec.PersistentVolume.Spec.Local == nil || len(spec.PersistentVolume.Spec.Local.Path) == 0 {
		return "", fmt.Errorf("local volume source is nil or local path is not set")
	}

	kvh, ok := plugin.host.(volume.KubeletVolumeHost)
	if !ok {
		return "", fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
	}

	fileType, err := kvh.GetHostUtil().GetFileType(spec.PersistentVolume.Spec.Local.Path)
	if err != nil {
		return "", err
	}
	switch fileType {
	case hostutil.FileTypeDirectory:
		return spec.PersistentVolume.Spec.Local.Path, nil
	case hostutil.FileTypeBlockDev:
		return filepath.Join(plugin.generateBlockDeviceBaseGlobalPath(), spec.Name()), nil
	default:
		return "", fmt.Errorf("only directory and block device are supported")
	}
}


CanDeviceMount

func (plugin *localVolumePlugin) CanDeviceMount(spec *volume.Spec) (bool, error) {
	return true, nil
}


NewDeviceMounter

func (plugin *localVolumePlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
	kvh, ok := plugin.host.(volume.KubeletVolumeHost)
	if !ok {
		return nil, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
	}
	return &deviceMounter{
		plugin:   plugin,
		mounter:  util.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host),
		hostUtil: kvh.GetHostUtil(),
	}, nil
}


mountLocalBlockDevice

func (dm *deviceMounter) mountLocalBlockDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error {
	klog.V(4).Infof("local: mounting device %s to %s", devicePath, deviceMountPath)
	notMnt, err := dm.mounter.IsLikelyNotMountPoint(deviceMountPath)
	if err != nil {
		if os.IsNotExist(err) {
			if err := os.MkdirAll(deviceMountPath, 0750); err != nil {
				return err
			}
			notMnt = true
		} else {
			return err
		}
	}
	if !notMnt {
		return nil
	}
	fstype, err := getVolumeSourceFSType(spec)
	if err != nil {
		return err
	}

	ro, err := getVolumeSourceReadOnly(spec)
	if err != nil {
		return err
	}
	options := []string{}
	if ro {
		options = append(options, "ro")
	}
	mountOptions := util.MountOptionFromSpec(spec, options...)
	err = dm.mounter.FormatAndMount(devicePath, deviceMountPath, fstype, mountOptions)
	if err != nil {
		if rmErr := os.Remove(deviceMountPath); rmErr != nil {
			klog.Warningf("local: failed to remove %s: %v", deviceMountPath, rmErr)
		}
		return fmt.Errorf("local: failed to mount device %s at %s (fstype: %s), error %w", devicePath, deviceMountPath, fstype, err)
	}
	klog.V(3).Infof("local: successfully mount device %s at %s (fstype: %s)", devicePath, deviceMountPath, fstype)
	return nil
}


MountDevice

func (dm *deviceMounter) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string, _ volume.DeviceMounterArgs) error {
	if spec.PersistentVolume.Spec.Local == nil || len(spec.PersistentVolume.Spec.Local.Path) == 0 {
		return fmt.Errorf("local volume source is nil or local path is not set")
	}
	fileType, err := dm.hostUtil.GetFileType(spec.PersistentVolume.Spec.Local.Path)
	if err != nil {
		return err
	}

	switch fileType {
	case hostutil.FileTypeBlockDev:
		// local volume plugin does not implement AttachableVolumePlugin interface, so set devicePath to Path in PV spec directly
		return dm.mountLocalBlockDevice(spec, spec.PersistentVolume.Spec.Local.Path, deviceMountPath)
	case hostutil.FileTypeDirectory:
		// if the given local volume path is of already filesystem directory, return directly
		return nil
	default:
		return fmt.Errorf("only directory and block device are supported")
	}
}


RequiresFSResize

func (plugin *localVolumePlugin) RequiresFSResize() bool {
	return true
}


NodeExpand

func (plugin *localVolumePlugin) NodeExpand(resizeOptions volume.NodeResizeOptions) (bool, error) {
	fsVolume, err := util.CheckVolumeModeFilesystem(resizeOptions.VolumeSpec)
	if err != nil {
		return false, fmt.Errorf("error checking VolumeMode: %v", err)
	}
	if !fsVolume {
		return true, nil
	}

	localDevicePath := resizeOptions.VolumeSpec.PersistentVolume.Spec.Local.Path

	kvh, ok := plugin.host.(volume.KubeletVolumeHost)
	if !ok {
		return false, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
	}

	fileType, err := kvh.GetHostUtil().GetFileType(localDevicePath)
	if err != nil {
		return false, err
	}

	switch fileType {
	case hostutil.FileTypeBlockDev:
		_, err = util.GenericResizeFS(plugin.host, plugin.GetPluginName(), localDevicePath, resizeOptions.DeviceMountPath)
		if err != nil {
			return false, err
		}
		return true, nil
	case hostutil.FileTypeDirectory:
		// if the given local volume path is of already filesystem directory, return directly because
		// we do not want to prevent mount operation from succeeding.
		klog.InfoS("Expansion of directory based local volumes is NO-OP", "localVolumePath", localDevicePath)
		return true, nil
	default:
		return false, fmt.Errorf("only directory and block device are supported")
	}
}


getVolumeSourceFSType

func getVolumeSourceFSType(spec *volume.Spec) (string, error) {
	if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.Local != nil {
		if spec.PersistentVolume.Spec.Local.FSType != nil {
			return *spec.PersistentVolume.Spec.Local.FSType, nil
		}
		// if the FSType is not set in local PV spec, setting it to default ("ext4")
		return defaultFSType, nil
	}

	return "", fmt.Errorf("spec does not reference a Local volume type")
}


getVolumeSourceReadOnly

func getVolumeSourceReadOnly(spec *volume.Spec) (bool, error) {
	if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.Local != nil {
		// local volumes used as a PersistentVolume gets the ReadOnly flag indirectly through
		// the persistent-claim volume used to mount the PV
		return spec.ReadOnly, nil
	}

	return false, fmt.Errorf("spec does not reference a Local volume type")
}


GetDeviceMountPath

func (dm *deviceMounter) GetDeviceMountPath(spec *volume.Spec) (string, error) {
	return dm.plugin.getGlobalLocalPath(spec)
}


NewDeviceUnmounter

func (plugin *localVolumePlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
	return &deviceMounter{
		plugin:  plugin,
		mounter: util.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host),
	}, nil
}


GetDeviceMountRefs

func (plugin *localVolumePlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
	mounter := plugin.host.GetMounter(plugin.GetPluginName())
	return mounter.GetMountRefs(deviceMountPath)
}


UnmountDevice

func (dm *deviceMounter) UnmountDevice(deviceMountPath string) error {
	// If the local PV is a block device,
	// The deviceMountPath is generated to the format like :/var/lib/kubelet/plugins/kubernetes.io/local-volume/mounts/localpv.spec.Name;
	// If it is a filesystem directory, then the deviceMountPath is set directly to pvSpec.Local.Path
	// We only need to unmount block device here, so we need to check if the deviceMountPath passed here
	// has base mount path: /var/lib/kubelet/plugins/kubernetes.io/local-volume/mounts
	basemountPath := dm.plugin.generateBlockDeviceBaseGlobalPath()
	if mount.PathWithinBase(deviceMountPath, basemountPath) {
		return mount.CleanupMountPoint(deviceMountPath, dm.mounter, false)
	}

	return nil
}


GetPath

func (l *localVolume) GetPath() string {
	return l.plugin.host.GetPodVolumeDir(l.podUID, utilstrings.EscapeQualifiedName(localVolumePluginName), l.volName)
}


GetAttributes

func (m *localVolumeMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:       m.readOnly,
		Managed:        !m.readOnly,
		SELinuxRelabel: true,
	}
}


SetUp

// SetUp bind mounts the directory to the volume path
func (m *localVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error {
	return m.SetUpAt(m.GetPath(), mounterArgs)
}


SetUpAt

// SetUpAt bind mounts the directory to the volume path and sets up volume ownership
func (m *localVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	m.plugin.volumeLocks.LockKey(m.globalPath)
	defer m.plugin.volumeLocks.UnlockKey(m.globalPath)

	if m.globalPath == "" {
		return fmt.Errorf("LocalVolume volume %q path is empty", m.volName)
	}

	err := validation.ValidatePathNoBacksteps(m.globalPath)
	if err != nil {
		return fmt.Errorf("invalid path: %s %v", m.globalPath, err)
	}

	notMnt, err := mount.IsNotMountPoint(m.mounter, dir)
	klog.V(4).Infof("LocalVolume mount setup: PodDir(%s) VolDir(%s) Mounted(%t) Error(%v), ReadOnly(%t)", dir, m.globalPath, !notMnt, err, m.readOnly)
	if err != nil && !os.IsNotExist(err) {
		klog.Errorf("cannot validate mount point: %s %v", dir, err)
		return err
	}

	if !notMnt {
		return nil
	}
	refs, err := m.mounter.GetMountRefs(m.globalPath)
	if mounterArgs.FsGroup != nil {
		if err != nil {
			klog.Errorf("cannot collect mounting information: %s %v", m.globalPath, err)
			return err
		}

		// Only count mounts from other pods
		refs = m.filterPodMounts(refs)
		if len(refs) > 0 {
			fsGroupNew := int64(*mounterArgs.FsGroup)
			_, fsGroupOld, err := m.hostUtil.GetOwner(m.globalPath)
			if err != nil {
				return fmt.Errorf("failed to check fsGroup for %s (%v)", m.globalPath, err)
			}
			if fsGroupNew != fsGroupOld {
				m.plugin.recorder.Eventf(m.pod, v1.EventTypeWarning, events.WarnAlreadyMountedVolume, "The requested fsGroup is %d, but the volume %s has GID %d. The volume may not be shareable.", fsGroupNew, m.volName, fsGroupOld)
			}
		}

	}

	if runtime.GOOS != "windows" {
		// skip below MkdirAll for windows since the "bind mount" logic is implemented differently in mount_wiondows.go
		if err := os.MkdirAll(dir, 0750); err != nil {
			klog.Errorf("mkdir failed on disk %s (%v)", dir, err)
			return err
		}
	}
	// Perform a bind mount to the full path to allow duplicate mounts of the same volume.
	options := []string{"bind"}
	if m.readOnly {
		options = append(options, "ro")
	}
	mountOptions := util.JoinMountOptions(options, m.mountOptions)

	klog.V(4).Infof("attempting to mount %s", dir)
	globalPath := util.MakeAbsolutePath(runtime.GOOS, m.globalPath)
	err = m.mounter.MountSensitiveWithoutSystemd(globalPath, dir, "", mountOptions, nil)
	if err != nil {
		klog.Errorf("Mount of volume %s failed: %v", dir, err)
		notMnt, mntErr := mount.IsNotMountPoint(m.mounter, dir)
		if mntErr != nil {
			klog.Errorf("IsNotMountPoint check failed: %v", mntErr)
			return err
		}
		if !notMnt {
			if mntErr = m.mounter.Unmount(dir); mntErr != nil {
				klog.Errorf("Failed to unmount: %v", mntErr)
				return err
			}
			notMnt, mntErr = mount.IsNotMountPoint(m.mounter, dir)
			if mntErr != nil {
				klog.Errorf("IsNotMountPoint check failed: %v", mntErr)
				return err
			}
			if !notMnt {
				// This is very odd, we don't expect it.  We'll try again next sync loop.
				klog.Errorf("%s is still mounted, despite call to unmount().  Will try again next sync loop.", dir)
				return err
			}
		}
		if rmErr := os.Remove(dir); rmErr != nil {
			klog.Warningf("failed to remove %s: %v", dir, rmErr)
		}
		return err
	}
	if !m.readOnly {
		// Volume owner will be written only once on the first volume mount
		if len(refs) == 0 {
			return volume.SetVolumeOwnership(m, dir, mounterArgs.FsGroup, mounterArgs.FSGroupChangePolicy, util.FSGroupCompleteHook(m.plugin, nil))
		}
	}
	return nil
}


filterPodMounts

// filterPodMounts only returns mount paths inside the kubelet pod directory
func (m *localVolumeMounter) filterPodMounts(refs []string) []string {
	filtered := []string{}
	for _, r := range refs {
		if strings.HasPrefix(r, m.plugin.host.GetPodsDir()+string(os.PathSeparator)) {
			filtered = append(filtered, r)
		}
	}
	return filtered
}


TearDown

// TearDown unmounts the bind mount
func (u *localVolumeUnmounter) TearDown() error {
	return u.TearDownAt(u.GetPath())
}


TearDownAt

// TearDownAt unmounts the bind mount
func (u *localVolumeUnmounter) TearDownAt(dir string) error {
	klog.V(4).Infof("Unmounting volume %q at path %q\n", u.volName, dir)
	return mount.CleanupMountPoint(dir, u.mounter, true) /* extensiveMountPointCheck = true */
}


SetUpDevice

// SetUpDevice prepares the volume to the node by the plugin specific way.
func (m *localVolumeMapper) SetUpDevice() (string, error) {
	return "", nil
}


MapPodDevice

// MapPodDevice provides physical device path for the local PV.
func (m *localVolumeMapper) MapPodDevice() (string, error) {
	globalPath := util.MakeAbsolutePath(runtime.GOOS, m.globalPath)
	klog.V(4).Infof("MapPodDevice returning path %s", globalPath)
	return globalPath, nil
}


GetStagingPath

// GetStagingPath returns
func (m *localVolumeMapper) GetStagingPath() string {
	return ""
}


SupportsMetrics

// SupportsMetrics returns true for SupportsMetrics as it initializes the
// MetricsProvider.
func (m *localVolumeMapper) SupportsMetrics() bool {
	return true
}


GetGlobalMapPath

// GetGlobalMapPath returns global map path and error.
// path: plugins/kubernetes.io/kubernetes.io/local-volume/volumeDevices/{volumeName}
func (l *localVolume) GetGlobalMapPath(spec *volume.Spec) (string, error) {
	return filepath.Join(l.plugin.host.GetVolumeDevicePluginDir(utilstrings.EscapeQualifiedName(localVolumePluginName)),
		l.volName), nil
}


GetPodDeviceMapPath

// GetPodDeviceMapPath returns pod device map path and volume name.
// path: pods/{podUid}/volumeDevices/kubernetes.io~local-volume
// volName: local-pv-ff0d6d4
func (l *localVolume) GetPodDeviceMapPath() (string, string) {
	return l.plugin.host.GetPodVolumeDeviceDir(l.podUID,
		utilstrings.EscapeQualifiedName(localVolumePluginName)), l.volName
}


NewMetricsBlock

// NewMetricsStatfs creates a new metricsBlock with the device node of the
// Volume.
func NewMetricsBlock(device string) MetricsProvider {
	return &metricsBlock{device}
}


GetMetrics

// See MetricsProvider.GetMetrics
// GetMetrics detects the size of the BlockMode volume for the device node
// where the Volume is attached.
//
// Note that only the capacity of the device can be detected with standard
// tools. Storage systems may have more information that they can provide by
// going through specialized APIs.
func (mb *metricsBlock) GetMetrics() (*Metrics, error) {
	startTime := time.Now()
	defer servermetrics.CollectVolumeStatCalDuration("block", startTime)

	// TODO: Windows does not yet support VolumeMode=Block
	if runtime.GOOS == "windows" {
		return nil, NewNotImplementedError("Windows does not support Block volumes")
	}

	metrics := &Metrics{Time: metav1.Now()}
	if mb.device == "" {
		return metrics, NewNoPathDefinedError()
	}

	err := mb.getBlockInfo(metrics)
	if err != nil {
		return metrics, err
	}

	return metrics, nil
}


getBlockInfo

// getBlockInfo fetches metrics.Capacity by opening the device and seeking to
// the end.
func (mb *metricsBlock) getBlockInfo(metrics *Metrics) error {
	dev, err := os.Open(mb.device)
	if err != nil {
		return fmt.Errorf("unable to open device %q: %w", mb.device, err)
	}
	defer dev.Close()

	end, err := dev.Seek(0, io.SeekEnd)
	if err != nil {
		return fmt.Errorf("failed to detect size of %q: %w", mb.device, err)
	}

	metrics.Capacity = resource.NewQuantity(end, resource.BinarySI)

	return nil
}


NewCachedMetrics

// NewCachedMetrics creates a new cachedMetrics wrapping another
// MetricsProvider and caching the results.
func NewCachedMetrics(provider MetricsProvider) MetricsProvider {
	return &cachedMetrics{wrapped: provider}
}


GetMetrics

// GetMetrics runs the wrapped metrics provider's GetMetrics method once and
// caches the result. Will not cache result if there is an error.
// See MetricsProvider.GetMetrics
func (md *cachedMetrics) GetMetrics() (*Metrics, error) {
	md.once.cache(func() error {
		md.resultMetrics, md.resultError = md.wrapped.GetMetrics()
		return md.resultError
	})
	return md.resultMetrics, md.resultError
}


cache

// Copied from sync.Once but we don't want to cache the results if there is an
// error
func (o *cacheOnce) cache(f func() error) {
	if atomic.LoadUint32(&o.done) == 1 {
		return
	}
	// Slow-path.
	o.m.Lock()
	defer o.m.Unlock()
	if o.done == 0 {
		err := f()
		if err == nil {
			atomic.StoreUint32(&o.done, 1)
		}
	}
}


NewMetricsDu

// NewMetricsDu creates a new metricsDu with the Volume path.
func NewMetricsDu(path string) MetricsProvider {
	return &metricsDu{path}
}


GetMetrics

// GetMetrics calculates the volume usage and device free space by executing "du"
// and gathering filesystem info for the Volume path.
// See MetricsProvider.GetMetrics
func (md *metricsDu) GetMetrics() (*Metrics, error) {
	metrics := &Metrics{Time: metav1.Now()}
	if md.path == "" {
		return metrics, NewNoPathDefinedError()
	}

	err := md.getDiskUsage(metrics)
	if err != nil {
		return metrics, err
	}

	err = md.getFsInfo(metrics)
	if err != nil {
		return metrics, err
	}

	return metrics, nil
}


getDiskUsage

// getDiskUsage writes metrics.Used and metric.InodesUsed from fs.DiskUsage
func (md *metricsDu) getDiskUsage(metrics *Metrics) error {
	usage, err := fs.DiskUsage(md.path)
	if err != nil {
		return err
	}
	metrics.Used = resource.NewQuantity(usage.Bytes, resource.BinarySI)
	metrics.InodesUsed = resource.NewQuantity(usage.Inodes, resource.BinarySI)
	return nil
}


getFsInfo

// getFsInfo writes metrics.Capacity and metrics.Available from the filesystem
// info
func (md *metricsDu) getFsInfo(metrics *Metrics) error {
	available, capacity, _, inodes, inodesFree, _, err := fs.Info(md.path)
	if err != nil {
		return NewFsInfoFailedError(err)
	}
	metrics.Available = resource.NewQuantity(available, resource.BinarySI)
	metrics.Capacity = resource.NewQuantity(capacity, resource.BinarySI)
	metrics.Inodes = resource.NewQuantity(inodes, resource.BinarySI)
	metrics.InodesFree = resource.NewQuantity(inodesFree, resource.BinarySI)
	return nil
}


NewNotSupportedError

// NewNotSupportedError creates a new MetricsError with code NotSupported.
func NewNotSupportedError() *MetricsError {
	return &MetricsError{
		Code: ErrCodeNotSupported,
		Msg:  "metrics are not supported for MetricsNil Volumes",
	}
}


NewNotImplementedError

// NewNotImplementedError creates a new MetricsError with code NotSupported.
func NewNotImplementedError(reason string) *MetricsError {
	return &MetricsError{
		Code: ErrCodeNotSupported,
		Msg:  fmt.Sprintf("metrics support is not implemented: %s", reason),
	}
}


NewNotSupportedErrorWithDriverName

// NewNotSupportedErrorWithDriverName creates a new MetricsError with code NotSupported.
// driver name is added to the error message.
func NewNotSupportedErrorWithDriverName(name string) *MetricsError {
	return &MetricsError{
		Code: ErrCodeNotSupported,
		Msg:  fmt.Sprintf("metrics are not supported for %s volumes", name),
	}
}


NewNoPathDefinedError

// NewNoPathDefinedError creates a new MetricsError with code NoPathDefined.
func NewNoPathDefinedError() *MetricsError {
	return &MetricsError{
		Code: ErrCodeNoPathDefined,
		Msg:  "no path defined for disk usage metrics.",
	}
}


NewFsInfoFailedError

// NewFsInfoFailedError creates a new MetricsError with code FsInfoFailed.
func NewFsInfoFailedError(err error) *MetricsError {
	return &MetricsError{
		Code: ErrCodeFsInfoFailed,
		Msg:  fmt.Sprintf("failed to get FsInfo due to error %v", err),
	}
}


Error

func (e *MetricsError) Error() string {
	return e.Msg
}


IsNotSupported

// IsNotSupported returns true if and only if err is "key" not found error.
func IsNotSupported(err error) bool {
	return isErrCode(err, ErrCodeNotSupported)
}


isErrCode

func isErrCode(err error, code int) bool {
	if err == nil {
		return false
	}
	if e, ok := err.(*MetricsError); ok {
		return e.Code == code
	}
	return false
}


SupportsMetrics

// SupportsMetrics returns false for the MetricsNil type.
func (*MetricsNil) SupportsMetrics() bool {
	return false
}


GetMetrics

// GetMetrics returns an empty Metrics and an error.
// See MetricsProvider.GetMetrics
func (*MetricsNil) GetMetrics() (*Metrics, error) {
	return &Metrics{}, NewNotSupportedError()
}


NewMetricsStatFS

// NewMetricsStatfs creates a new metricsStatFS with the Volume path.
func NewMetricsStatFS(path string) MetricsProvider {
	return &metricsStatFS{path}
}


GetMetrics

// See MetricsProvider.GetMetrics
// GetMetrics calculates the volume usage and device free space by executing "du"
// and gathering filesystem info for the Volume path.
func (md *metricsStatFS) GetMetrics() (*Metrics, error) {
	startTime := time.Now()
	defer servermetrics.CollectVolumeStatCalDuration("statfs", startTime)

	metrics := &Metrics{Time: metav1.Now()}
	if md.path == "" {
		return metrics, NewNoPathDefinedError()
	}

	err := md.getFsInfo(metrics)
	if err != nil {
		return metrics, err
	}

	return metrics, nil
}


getFsInfo

// getFsInfo writes metrics.Capacity, metrics.Used and metrics.Available from the filesystem info
func (md *metricsStatFS) getFsInfo(metrics *Metrics) error {
	available, capacity, usage, inodes, inodesFree, inodesUsed, err := fs.Info(md.path)
	if err != nil {
		return NewFsInfoFailedError(err)
	}
	metrics.Available = resource.NewQuantity(available, resource.BinarySI)
	metrics.Capacity = resource.NewQuantity(capacity, resource.BinarySI)
	metrics.Used = resource.NewQuantity(usage, resource.BinarySI)
	metrics.Inodes = resource.NewQuantity(inodes, resource.BinarySI)
	metrics.InodesFree = resource.NewQuantity(inodesFree, resource.BinarySI)
	metrics.InodesUsed = resource.NewQuantity(inodesUsed, resource.BinarySI)
	return nil
}


getPath

func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
	return host.GetPodVolumeDir(uid, utilstrings.EscapeQualifiedName(nfsPluginName), volName)
}


ProbeVolumePlugins

// ProbeVolumePlugins is the primary entrypoint for volume plugins.
// This is the primary entrypoint for volume plugins.
// The volumeConfig arg provides the ability to configure recycler behavior.  It is implemented as a pointer to allow nils.
// The nfsPlugin is used to store the volumeConfig and give it, when needed, to the func that creates NFS Recyclers.
// Tests that exercise recycling should not use this func but instead use ProbeRecyclablePlugins() to override default behavior.
func ProbeVolumePlugins(volumeConfig volume.VolumeConfig) []volume.VolumePlugin {
	return []volume.VolumePlugin{
		&nfsPlugin{
			host:   nil,
			config: volumeConfig,
		},
	}
}


Init

func (plugin *nfsPlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	return nil
}


GetPluginName

func (plugin *nfsPlugin) GetPluginName() string {
	return nfsPluginName
}


GetVolumeName

func (plugin *nfsPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf(
		"%v/%v",
		volumeSource.Server,
		volumeSource.Path), nil
}


CanSupport

func (plugin *nfsPlugin) CanSupport(spec *volume.Spec) bool {
	return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.NFS != nil) ||
		(spec.Volume != nil && spec.Volume.NFS != nil)
}


RequiresRemount

func (plugin *nfsPlugin) RequiresRemount(spec *volume.Spec) bool {
	return false
}


SupportsMountOption

func (plugin *nfsPlugin) SupportsMountOption() bool {
	return true
}


SupportsBulkVolumeVerification

func (plugin *nfsPlugin) SupportsBulkVolumeVerification() bool {
	return false
}


SupportsSELinuxContextMount

func (plugin *nfsPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return false, nil
}


GetAccessModes

func (plugin *nfsPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
	return []v1.PersistentVolumeAccessMode{
		v1.ReadWriteOnce,
		v1.ReadOnlyMany,
		v1.ReadWriteMany,
	}
}


NewMounter

func (plugin *nfsPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
	return plugin.newMounterInternal(spec, pod, plugin.host.GetMounter(plugin.GetPluginName()))
}


newMounterInternal

func (plugin *nfsPlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod, mounter mount.Interface) (volume.Mounter, error) {
	source, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}
	return &nfsMounter{
		nfs: &nfs{
			volName:         spec.Name(),
			mounter:         mounter,
			pod:             pod,
			plugin:          plugin,
			MetricsProvider: volume.NewMetricsStatFS(getPath(pod.UID, spec.Name(), plugin.host)),
		},
		server:       getServerFromSource(source),
		exportPath:   source.Path,
		readOnly:     readOnly,
		mountOptions: util.MountOptionFromSpec(spec),
	}, nil
}


NewUnmounter

func (plugin *nfsPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter(plugin.GetPluginName()))
}


newUnmounterInternal

func (plugin *nfsPlugin) newUnmounterInternal(volName string, podUID types.UID, mounter mount.Interface) (volume.Unmounter, error) {
	return &nfsUnmounter{&nfs{
		volName:         volName,
		mounter:         mounter,
		pod:             &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: podUID}},
		plugin:          plugin,
		MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)),
	}}, nil
}


Recycle

// Recycle recycles/scrubs clean an NFS volume.
// Recycle blocks until the pod has completed or any error occurs.
func (plugin *nfsPlugin) Recycle(pvName string, spec *volume.Spec, eventRecorder recyclerclient.RecycleEventRecorder) error {
	if spec.PersistentVolume == nil || spec.PersistentVolume.Spec.NFS == nil {
		return fmt.Errorf("spec.PersistentVolumeSource.NFS is nil")
	}

	pod := plugin.config.RecyclerPodTemplate
	timeout := util.CalculateTimeoutForVolume(plugin.config.RecyclerMinimumTimeout, plugin.config.RecyclerTimeoutIncrement, spec.PersistentVolume)
	// overrides
	pod.Spec.ActiveDeadlineSeconds = &timeout
	pod.GenerateName = "pv-recycler-nfs-"
	pod.Spec.Volumes[0].VolumeSource = v1.VolumeSource{
		NFS: &v1.NFSVolumeSource{
			Server: spec.PersistentVolume.Spec.NFS.Server,
			Path:   spec.PersistentVolume.Spec.NFS.Path,
		},
	}
	return recyclerclient.RecycleVolumeByWatchingPodUntilCompletion(pvName, pod, plugin.host.GetKubeClient(), eventRecorder)
}


ConstructVolumeSpec

func (plugin *nfsPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
	nfsVolume := &v1.Volume{
		Name: volumeName,
		VolumeSource: v1.VolumeSource{
			NFS: &v1.NFSVolumeSource{
				Path: volumeName,
			},
		},
	}
	return volume.ReconstructedVolume{
		Spec: volume.NewSpecFromVolume(nfsVolume),
	}, nil
}


GetPath

func (nfsVolume *nfs) GetPath() string {
	name := nfsPluginName
	return nfsVolume.plugin.host.GetPodVolumeDir(nfsVolume.pod.UID, utilstrings.EscapeQualifiedName(name), nfsVolume.volName)
}


GetAttributes

func (nfsMounter *nfsMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:       nfsMounter.readOnly,
		Managed:        false,
		SELinuxRelabel: false,
	}
}


SetUp

// SetUp attaches the disk and bind mounts to the volume path.
func (nfsMounter *nfsMounter) SetUp(mounterArgs volume.MounterArgs) error {
	return nfsMounter.SetUpAt(nfsMounter.GetPath(), mounterArgs)
}


SetUpAt

func (nfsMounter *nfsMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	notMnt, err := mount.IsNotMountPoint(nfsMounter.mounter, dir)
	klog.V(4).Infof("NFS mount set up: %s %v %v", dir, !notMnt, err)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	if !notMnt {
		return nil
	}
	if err := os.MkdirAll(dir, 0750); err != nil {
		return err
	}
	source := fmt.Sprintf("%s:%s", nfsMounter.server, nfsMounter.exportPath)
	options := []string{}
	if nfsMounter.readOnly {
		options = append(options, "ro")
	}
	mountOptions := util.JoinMountOptions(nfsMounter.mountOptions, options)
	err = nfsMounter.mounter.MountSensitiveWithoutSystemd(source, dir, "nfs", mountOptions, nil)
	if err != nil {
		notMnt, mntErr := mount.IsNotMountPoint(nfsMounter.mounter, dir)
		if mntErr != nil {
			klog.Errorf("IsNotMountPoint check failed: %v", mntErr)
			return err
		}
		if !notMnt {
			if mntErr = nfsMounter.mounter.Unmount(dir); mntErr != nil {
				klog.Errorf("Failed to unmount: %v", mntErr)
				return err
			}
			notMnt, mntErr := mount.IsNotMountPoint(nfsMounter.mounter, dir)
			if mntErr != nil {
				klog.Errorf("IsNotMountPoint check failed: %v", mntErr)
				return err
			}
			if !notMnt {
				// This is very odd, we don't expect it.  We'll try again next sync loop.
				klog.Errorf("%s is still mounted, despite call to unmount().  Will try again next sync loop.", dir)
				return err
			}
		}
		os.Remove(dir)
		return err
	}
	return nil
}


TearDown

func (c *nfsUnmounter) TearDown() error {
	return c.TearDownAt(c.GetPath())
}


TearDownAt

func (c *nfsUnmounter) TearDownAt(dir string) error {
	// Use extensiveMountPointCheck to consult /proc/mounts. We can't use faster
	// IsLikelyNotMountPoint (lstat()), since there may be root_squash on the
	// NFS server and kubelet may not be able to do lstat/stat() there.
	forceUnmounter, ok := c.mounter.(mount.MounterForceUnmounter)
	if ok {
		klog.V(4).Infof("Using force unmounter interface")
		return mount.CleanupMountWithForce(dir, forceUnmounter, true /* extensiveMountPointCheck */, unMountTimeout)
	}
	return mount.CleanupMountPoint(dir, c.mounter, true /* extensiveMountPointCheck */)
}


getVolumeSource

func getVolumeSource(spec *volume.Spec) (*v1.NFSVolumeSource, bool, error) {
	if spec.Volume != nil && spec.Volume.NFS != nil {
		return spec.Volume.NFS, spec.Volume.NFS.ReadOnly, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.NFS != nil {
		return spec.PersistentVolume.Spec.NFS, spec.ReadOnly, nil
	}

	return nil, false, fmt.Errorf("Spec does not reference a NFS volume type")
}


getServerFromSource

func getServerFromSource(source *v1.NFSVolumeSource) string {
	if netutil.IsIPv6String(source.Server) {
		return fmt.Sprintf("[%s]", source.Server)
	}
	return source.Server
}


ExpandVolumeDevice

func (n *noopExpandableVolumePluginInstance) ExpandVolumeDevice(spec *Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) {
	return newSize, nil
}


Init

func (n *noopExpandableVolumePluginInstance) Init(host VolumeHost) error {
	return nil
}


GetPluginName

func (n *noopExpandableVolumePluginInstance) GetPluginName() string {
	return n.spec.KubeletExpandablePluginName()
}


GetVolumeName

func (n *noopExpandableVolumePluginInstance) GetVolumeName(spec *Spec) (string, error) {
	return n.spec.Name(), nil
}


CanSupport

func (n *noopExpandableVolumePluginInstance) CanSupport(spec *Spec) bool {
	return true
}


RequiresRemount

func (n *noopExpandableVolumePluginInstance) RequiresRemount(spec *Spec) bool {
	return false
}


NewMounter

func (n *noopExpandableVolumePluginInstance) NewMounter(spec *Spec, podRef *v1.Pod, opts VolumeOptions) (Mounter, error) {
	return nil, nil
}


NewUnmounter

func (n *noopExpandableVolumePluginInstance) NewUnmounter(name string, podUID types.UID) (Unmounter, error) {
	return nil, nil
}


ConstructVolumeSpec

func (n *noopExpandableVolumePluginInstance) ConstructVolumeSpec(volumeName, mountPath string) (ReconstructedVolume, error) {
	return ReconstructedVolume{Spec: n.spec}, nil
}


SupportsMountOption

func (n *noopExpandableVolumePluginInstance) SupportsMountOption() bool {
	return true
}


SupportsBulkVolumeVerification

func (n *noopExpandableVolumePluginInstance) SupportsBulkVolumeVerification() bool {
	return false
}


RequiresFSResize

func (n *noopExpandableVolumePluginInstance) RequiresFSResize() bool {
	return true
}


SupportsSELinuxContextMount

func (n *noopExpandableVolumePluginInstance) SupportsSELinuxContextMount(spec *Spec) (bool, error) {
	return false, nil
}


Name

// Name returns the name of either Volume or PersistentVolume, one of which must not be nil.
func (spec *Spec) Name() string {
	switch {
	case spec.Volume != nil:
		return spec.Volume.Name
	case spec.PersistentVolume != nil:
		return spec.PersistentVolume.Name
	default:
		return ""
	}
}


IsKubeletExpandable

// IsKubeletExpandable returns true for volume types that can be expanded only by the node
// and not the controller. Currently Flex volume is the only one in this category since
// it is typically not installed on the controller
func (spec *Spec) IsKubeletExpandable() bool {
	switch {
	case spec.Volume != nil:
		return spec.Volume.FlexVolume != nil
	case spec.PersistentVolume != nil:
		return spec.PersistentVolume.Spec.FlexVolume != nil
	default:
		return false
	}
}


KubeletExpandablePluginName

// KubeletExpandablePluginName creates and returns a name for the plugin
// this is used in context on the controller where the plugin lookup fails
// as volume expansion on controller isn't supported, but a plugin name is
// required
func (spec *Spec) KubeletExpandablePluginName() string {
	switch {
	case spec.Volume != nil && spec.Volume.FlexVolume != nil:
		return spec.Volume.FlexVolume.Driver
	case spec.PersistentVolume != nil && spec.PersistentVolume.Spec.FlexVolume != nil:
		return spec.PersistentVolume.Spec.FlexVolume.Driver
	default:
		return ""
	}
}


NewSpecFromVolume

// NewSpecFromVolume creates an Spec from an v1.Volume
func NewSpecFromVolume(vs *v1.Volume) *Spec {
	return &Spec{
		Volume: vs,
	}
}


NewSpecFromPersistentVolume

// NewSpecFromPersistentVolume creates an Spec from an v1.PersistentVolume
func NewSpecFromPersistentVolume(pv *v1.PersistentVolume, readOnly bool) *Spec {
	return &Spec{
		PersistentVolume: pv,
		ReadOnly:         readOnly,
	}
}


InitPlugins

// InitPlugins initializes each plugin.  All plugins must have unique names.
// This must be called exactly once before any New* methods are called on any
// plugins.
func (pm *VolumePluginMgr) InitPlugins(plugins []VolumePlugin, prober DynamicPluginProber, host VolumeHost) error {
	pm.mutex.Lock()
	defer pm.mutex.Unlock()

	pm.Host = host
	pm.loggedDeprecationWarnings = sets.NewString()

	if prober == nil {
		// Use a dummy prober to prevent nil deference.
		pm.prober = &dummyPluginProber{}
	} else {
		pm.prober = prober
	}
	if err := pm.prober.Init(); err != nil {
		// Prober init failure should not affect the initialization of other plugins.
		klog.ErrorS(err, "Error initializing dynamic plugin prober")
		pm.prober = &dummyPluginProber{}
	}

	if pm.plugins == nil {
		pm.plugins = map[string]VolumePlugin{}
	}
	if pm.probedPlugins == nil {
		pm.probedPlugins = map[string]VolumePlugin{}
	}

	allErrs := []error{}
	for _, plugin := range plugins {
		name := plugin.GetPluginName()
		if errs := validation.IsQualifiedName(name); len(errs) != 0 {
			allErrs = append(allErrs, fmt.Errorf("volume plugin has invalid name: %q: %s", name, strings.Join(errs, ";")))
			continue
		}

		if _, found := pm.plugins[name]; found {
			allErrs = append(allErrs, fmt.Errorf("volume plugin %q was registered more than once", name))
			continue
		}
		err := plugin.Init(host)
		if err != nil {
			klog.ErrorS(err, "Failed to load volume plugin", "pluginName", name)
			allErrs = append(allErrs, err)
			continue
		}
		pm.plugins[name] = plugin
		klog.V(1).InfoS("Loaded volume plugin", "pluginName", name)
	}
	return utilerrors.NewAggregate(allErrs)
}


initProbedPlugin

func (pm *VolumePluginMgr) initProbedPlugin(probedPlugin VolumePlugin) error {
	name := probedPlugin.GetPluginName()
	if errs := validation.IsQualifiedName(name); len(errs) != 0 {
		return fmt.Errorf("volume plugin has invalid name: %q: %s", name, strings.Join(errs, ";"))
	}

	err := probedPlugin.Init(pm.Host)
	if err != nil {
		return fmt.Errorf("failed to load volume plugin %s, error: %s", name, err.Error())
	}

	klog.V(1).InfoS("Loaded volume plugin", "pluginName", name)
	return nil
}


FindPluginBySpec

// FindPluginBySpec looks for a plugin that can support a given volume
// specification.  If no plugins can support or more than one plugin can
// support it, return error.
func (pm *VolumePluginMgr) FindPluginBySpec(spec *Spec) (VolumePlugin, error) {
	pm.mutex.RLock()
	defer pm.mutex.RUnlock()

	if spec == nil {
		return nil, fmt.Errorf("could not find plugin because volume spec is nil")
	}

	var match VolumePlugin
	matchedPluginNames := []string{}
	for _, v := range pm.plugins {
		if v.CanSupport(spec) {
			match = v
			matchedPluginNames = append(matchedPluginNames, v.GetPluginName())
		}
	}

	pm.refreshProbedPlugins()
	for _, plugin := range pm.probedPlugins {
		if plugin.CanSupport(spec) {
			match = plugin
			matchedPluginNames = append(matchedPluginNames, plugin.GetPluginName())
		}
	}

	if len(matchedPluginNames) == 0 {
		return nil, fmt.Errorf("no volume plugin matched")
	}
	if len(matchedPluginNames) > 1 {
		return nil, fmt.Errorf("multiple volume plugins matched: %s", strings.Join(matchedPluginNames, ","))
	}

	return match, nil
}


FindPluginByName

// FindPluginByName fetches a plugin by name or by legacy name.  If no plugin
// is found, returns error.
func (pm *VolumePluginMgr) FindPluginByName(name string) (VolumePlugin, error) {
	pm.mutex.RLock()
	defer pm.mutex.RUnlock()

	// Once we can get rid of legacy names we can reduce this to a map lookup.
	var match VolumePlugin
	if v, found := pm.plugins[name]; found {
		match = v
	}

	pm.refreshProbedPlugins()
	if plugin, found := pm.probedPlugins[name]; found {
		if match != nil {
			return nil, fmt.Errorf("multiple volume plugins matched: %s and %s", match.GetPluginName(), plugin.GetPluginName())
		}
		match = plugin
	}

	if match == nil {
		return nil, fmt.Errorf("no volume plugin matched name: %s", name)
	}
	return match, nil
}


refreshProbedPlugins

// Check if probedPlugin cache update is required.
// If it is, initialize all probed plugins and replace the cache with them.
func (pm *VolumePluginMgr) refreshProbedPlugins() {
	events, err := pm.prober.Probe()

	if err != nil {
		klog.ErrorS(err, "Error dynamically probing plugins")
	}

	// because the probe function can return a list of valid plugins
	// even when an error is present we still must add the plugins
	// or they will be skipped because each event only fires once
	for _, event := range events {
		if event.Op == ProbeAddOrUpdate {
			if err := pm.initProbedPlugin(event.Plugin); err != nil {
				klog.ErrorS(err, "Error initializing dynamically probed plugin",
					"pluginName", event.Plugin.GetPluginName())
				continue
			}
			pm.probedPlugins[event.Plugin.GetPluginName()] = event.Plugin
		} else if event.Op == ProbeRemove {
			// Plugin is not available on ProbeRemove event, only PluginName
			delete(pm.probedPlugins, event.PluginName)
		} else {
			klog.ErrorS(nil, "Unknown Operation on PluginName.",
				"pluginName", event.Plugin.GetPluginName())
		}
	}
}


ListVolumePluginWithLimits

// ListVolumePluginWithLimits returns plugins that have volume limits on nodes
func (pm *VolumePluginMgr) ListVolumePluginWithLimits() []VolumePluginWithAttachLimits {
	pm.mutex.RLock()
	defer pm.mutex.RUnlock()

	matchedPlugins := []VolumePluginWithAttachLimits{}
	for _, v := range pm.plugins {
		if plugin, ok := v.(VolumePluginWithAttachLimits); ok {
			matchedPlugins = append(matchedPlugins, plugin)
		}
	}
	return matchedPlugins
}


FindPersistentPluginBySpec

// FindPersistentPluginBySpec looks for a persistent volume plugin that can
// support a given volume specification.  If no plugin is found, return an
// error
func (pm *VolumePluginMgr) FindPersistentPluginBySpec(spec *Spec) (PersistentVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		return nil, fmt.Errorf("could not find volume plugin for spec: %#v", spec)
	}
	if persistentVolumePlugin, ok := volumePlugin.(PersistentVolumePlugin); ok {
		return persistentVolumePlugin, nil
	}
	return nil, fmt.Errorf("no persistent volume plugin matched")
}


FindVolumePluginWithLimitsBySpec

// FindVolumePluginWithLimitsBySpec returns volume plugin that has a limit on how many
// of them can be attached to a node
func (pm *VolumePluginMgr) FindVolumePluginWithLimitsBySpec(spec *Spec) (VolumePluginWithAttachLimits, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		return nil, fmt.Errorf("could not find volume plugin for spec : %#v", spec)
	}

	if limitedPlugin, ok := volumePlugin.(VolumePluginWithAttachLimits); ok {
		return limitedPlugin, nil
	}
	return nil, fmt.Errorf("no plugin with limits found")
}


FindPersistentPluginByName

// FindPersistentPluginByName fetches a persistent volume plugin by name.  If
// no plugin is found, returns error.
func (pm *VolumePluginMgr) FindPersistentPluginByName(name string) (PersistentVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginByName(name)
	if err != nil {
		return nil, err
	}
	if persistentVolumePlugin, ok := volumePlugin.(PersistentVolumePlugin); ok {
		return persistentVolumePlugin, nil
	}
	return nil, fmt.Errorf("no persistent volume plugin matched")
}


FindRecyclablePluginBySpec

// FindRecyclablePluginByName fetches a persistent volume plugin by name.  If
// no plugin is found, returns error.
func (pm *VolumePluginMgr) FindRecyclablePluginBySpec(spec *Spec) (RecyclableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		return nil, err
	}
	if recyclableVolumePlugin, ok := volumePlugin.(RecyclableVolumePlugin); ok {
		return recyclableVolumePlugin, nil
	}
	return nil, fmt.Errorf("no recyclable volume plugin matched")
}


FindProvisionablePluginByName

// FindProvisionablePluginByName fetches  a persistent volume plugin by name.  If
// no plugin is found, returns error.
func (pm *VolumePluginMgr) FindProvisionablePluginByName(name string) (ProvisionableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginByName(name)
	if err != nil {
		return nil, err
	}
	if provisionableVolumePlugin, ok := volumePlugin.(ProvisionableVolumePlugin); ok {
		return provisionableVolumePlugin, nil
	}
	return nil, fmt.Errorf("no provisionable volume plugin matched")
}


FindDeletablePluginBySpec

// FindDeletablePluginBySpec fetches a persistent volume plugin by spec.  If
// no plugin is found, returns error.
func (pm *VolumePluginMgr) FindDeletablePluginBySpec(spec *Spec) (DeletableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		return nil, err
	}
	if deletableVolumePlugin, ok := volumePlugin.(DeletableVolumePlugin); ok {
		return deletableVolumePlugin, nil
	}
	return nil, fmt.Errorf("no deletable volume plugin matched")
}


FindDeletablePluginByName

// FindDeletablePluginByName fetches a persistent volume plugin by name.  If
// no plugin is found, returns error.
func (pm *VolumePluginMgr) FindDeletablePluginByName(name string) (DeletableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginByName(name)
	if err != nil {
		return nil, err
	}
	if deletableVolumePlugin, ok := volumePlugin.(DeletableVolumePlugin); ok {
		return deletableVolumePlugin, nil
	}
	return nil, fmt.Errorf("no deletable volume plugin matched")
}


FindCreatablePluginBySpec

// FindCreatablePluginBySpec fetches a persistent volume plugin by name.  If
// no plugin is found, returns error.
func (pm *VolumePluginMgr) FindCreatablePluginBySpec(spec *Spec) (ProvisionableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		return nil, err
	}
	if provisionableVolumePlugin, ok := volumePlugin.(ProvisionableVolumePlugin); ok {
		return provisionableVolumePlugin, nil
	}
	return nil, fmt.Errorf("no creatable volume plugin matched")
}


FindAttachablePluginBySpec

// FindAttachablePluginBySpec fetches a persistent volume plugin by spec.
// Unlike the other "FindPlugin" methods, this does not return error if no
// plugin is found.  All volumes require a mounter and unmounter, but not
// every volume will have an attacher/detacher.
func (pm *VolumePluginMgr) FindAttachablePluginBySpec(spec *Spec) (AttachableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		return nil, err
	}
	if attachableVolumePlugin, ok := volumePlugin.(AttachableVolumePlugin); ok {
		if canAttach, err := attachableVolumePlugin.CanAttach(spec); err != nil {
			return nil, err
		} else if canAttach {
			return attachableVolumePlugin, nil
		}
	}
	return nil, nil
}


FindAttachablePluginByName

// FindAttachablePluginByName fetches an attachable volume plugin by name.
// Unlike the other "FindPlugin" methods, this does not return error if no
// plugin is found.  All volumes require a mounter and unmounter, but not
// every volume will have an attacher/detacher.
func (pm *VolumePluginMgr) FindAttachablePluginByName(name string) (AttachableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginByName(name)
	if err != nil {
		return nil, err
	}
	if attachablePlugin, ok := volumePlugin.(AttachableVolumePlugin); ok {
		return attachablePlugin, nil
	}
	return nil, nil
}


FindDeviceMountablePluginBySpec

// FindDeviceMountablePluginBySpec fetches a persistent volume plugin by spec.
func (pm *VolumePluginMgr) FindDeviceMountablePluginBySpec(spec *Spec) (DeviceMountableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		return nil, err
	}
	if deviceMountableVolumePlugin, ok := volumePlugin.(DeviceMountableVolumePlugin); ok {
		if canMount, err := deviceMountableVolumePlugin.CanDeviceMount(spec); err != nil {
			return nil, err
		} else if canMount {
			return deviceMountableVolumePlugin, nil
		}
	}
	return nil, nil
}


FindDeviceMountablePluginByName

// FindDeviceMountablePluginByName fetches a devicemountable volume plugin by name.
func (pm *VolumePluginMgr) FindDeviceMountablePluginByName(name string) (DeviceMountableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginByName(name)
	if err != nil {
		return nil, err
	}
	if deviceMountableVolumePlugin, ok := volumePlugin.(DeviceMountableVolumePlugin); ok {
		return deviceMountableVolumePlugin, nil
	}
	return nil, nil
}


FindExpandablePluginBySpec

// FindExpandablePluginBySpec fetches a persistent volume plugin by spec.
func (pm *VolumePluginMgr) FindExpandablePluginBySpec(spec *Spec) (ExpandableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		if spec.IsKubeletExpandable() {
			// for kubelet expandable volumes, return a noop plugin that
			// returns success for expand on the controller
			klog.V(4).InfoS("FindExpandablePluginBySpec -> returning noopExpandableVolumePluginInstance", "specName", spec.Name())
			return &noopExpandableVolumePluginInstance{spec}, nil
		}
		klog.V(4).InfoS("FindExpandablePluginBySpec -> err", "specName", spec.Name(), "err", err)
		return nil, err
	}

	if expandableVolumePlugin, ok := volumePlugin.(ExpandableVolumePlugin); ok {
		return expandableVolumePlugin, nil
	}
	return nil, nil
}


FindExpandablePluginByName

// FindExpandablePluginBySpec fetches a persistent volume plugin by name.
func (pm *VolumePluginMgr) FindExpandablePluginByName(name string) (ExpandableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginByName(name)
	if err != nil {
		return nil, err
	}

	if expandableVolumePlugin, ok := volumePlugin.(ExpandableVolumePlugin); ok {
		return expandableVolumePlugin, nil
	}
	return nil, nil
}


FindMapperPluginBySpec

// FindMapperPluginBySpec fetches a block volume plugin by spec.
func (pm *VolumePluginMgr) FindMapperPluginBySpec(spec *Spec) (BlockVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		return nil, err
	}

	if blockVolumePlugin, ok := volumePlugin.(BlockVolumePlugin); ok {
		return blockVolumePlugin, nil
	}
	return nil, nil
}


FindMapperPluginByName

// FindMapperPluginByName fetches a block volume plugin by name.
func (pm *VolumePluginMgr) FindMapperPluginByName(name string) (BlockVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginByName(name)
	if err != nil {
		return nil, err
	}

	if blockVolumePlugin, ok := volumePlugin.(BlockVolumePlugin); ok {
		return blockVolumePlugin, nil
	}
	return nil, nil
}


FindNodeExpandablePluginBySpec

// FindNodeExpandablePluginBySpec fetches a persistent volume plugin by spec
func (pm *VolumePluginMgr) FindNodeExpandablePluginBySpec(spec *Spec) (NodeExpandableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginBySpec(spec)
	if err != nil {
		return nil, err
	}
	if fsResizablePlugin, ok := volumePlugin.(NodeExpandableVolumePlugin); ok {
		return fsResizablePlugin, nil
	}
	return nil, nil
}


FindNodeExpandablePluginByName

// FindNodeExpandablePluginByName fetches a persistent volume plugin by name
func (pm *VolumePluginMgr) FindNodeExpandablePluginByName(name string) (NodeExpandableVolumePlugin, error) {
	volumePlugin, err := pm.FindPluginByName(name)
	if err != nil {
		return nil, err
	}

	if fsResizablePlugin, ok := volumePlugin.(NodeExpandableVolumePlugin); ok {
		return fsResizablePlugin, nil
	}

	return nil, nil
}


Run

func (pm *VolumePluginMgr) Run(stopCh <-chan struct{}) {
	kletHost, ok := pm.Host.(KubeletVolumeHost)
	if ok {
		// start informer for CSIDriver
		informerFactory := kletHost.GetInformerFactory()
		informerFactory.Start(stopCh)
		informerFactory.WaitForCacheSync(stopCh)
	}
}


NewPersistentVolumeRecyclerPodTemplate

// NewPersistentVolumeRecyclerPodTemplate creates a template for a recycler
// pod.  By default, a recycler pod simply runs "rm -rf" on a volume and tests
// for emptiness.  Most attributes of the template will be correct for most
// plugin implementations.  The following attributes can be overridden per
// plugin via configuration:
//
//  1. pod.Spec.Volumes[0].VolumeSource must be overridden.  Recycler
//     implementations without a valid VolumeSource will fail.
//  2. pod.GenerateName helps distinguish recycler pods by name.  Recommended.
//     Default is "pv-recycler-".
//  3. pod.Spec.ActiveDeadlineSeconds gives the recycler pod a maximum timeout
//     before failing.  Recommended.  Default is 60 seconds.
//
// See HostPath and NFS for working recycler examples
func NewPersistentVolumeRecyclerPodTemplate() *v1.Pod {
	timeout := int64(60)
	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "pv-recycler-",
			Namespace:    metav1.NamespaceDefault,
		},
		Spec: v1.PodSpec{
			ActiveDeadlineSeconds: &timeout,
			RestartPolicy:         v1.RestartPolicyNever,
			Volumes: []v1.Volume{
				{
					Name: "vol",
					// IMPORTANT!  All plugins using this template MUST
					// override pod.Spec.Volumes[0].VolumeSource Recycler
					// implementations without a valid VolumeSource will fail.
					VolumeSource: v1.VolumeSource{},
				},
			},
			Containers: []v1.Container{
				{
					Name:    "pv-recycler",
					Image:   "registry.k8s.io/debian-base:v2.0.0",
					Command: []string{"/bin/sh"},
					Args:    []string{"-c", "test -e /scrub && rm -rf /scrub/..?* /scrub/.[!.]* /scrub/*  && test -z \"$(ls -A /scrub)\" || exit 1"},
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      "vol",
							MountPath: "/scrub",
						},
					},
				},
			},
		},
	}
	return pod
}


ValidateRecyclerPodTemplate

// Check validity of recycle pod template
// List of checks:
// - at least one volume is defined in the recycle pod template
// If successful, returns nil
// if unsuccessful, returns an error.
func ValidateRecyclerPodTemplate(pod *v1.Pod) error {
	if len(pod.Spec.Volumes) < 1 {
		return fmt.Errorf("does not contain any volume(s)")
	}
	return nil
}


Init

func (*dummyPluginProber) Init() error                  { return nil }


Probe

func (*dummyPluginProber) Probe() ([]ProbeEvent, error) { return nil, nil }


ProbeVolumePlugins

// ProbeVolumePlugins is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&portworxVolumePlugin{nil, nil}}
}


getPath

func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
	return host.GetPodVolumeDir(uid, utilstrings.EscapeQualifiedName(portworxVolumePluginName), volName)
}


IsMigratedToCSI

func (plugin *portworxVolumePlugin) IsMigratedToCSI() bool {
	return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationPortworx)
}


Init

func (plugin *portworxVolumePlugin) Init(host volume.VolumeHost) error {
	client, err := volumeclient.NewDriverClient(
		fmt.Sprintf("http://%s", net.JoinHostPort(host.GetHostName(), strconv.Itoa(osdMgmtDefaultPort))),
		pxdDriverName, osdDriverVersion, pxDriverName)
	if err != nil {
		return err
	}

	plugin.host = host
	plugin.util = &portworxVolumeUtil{
		portworxClient: client,
	}

	return nil
}


GetPluginName

func (plugin *portworxVolumePlugin) GetPluginName() string {
	return portworxVolumePluginName
}


GetVolumeName

func (plugin *portworxVolumePlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return volumeSource.VolumeID, nil
}


CanSupport

func (plugin *portworxVolumePlugin) CanSupport(spec *volume.Spec) bool {
	return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.PortworxVolume != nil) ||
		(spec.Volume != nil && spec.Volume.PortworxVolume != nil)
}


RequiresRemount

func (plugin *portworxVolumePlugin) RequiresRemount(spec *volume.Spec) bool {
	return false
}


GetAccessModes

func (plugin *portworxVolumePlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
	return []v1.PersistentVolumeAccessMode{
		v1.ReadWriteOnce,
		v1.ReadWriteMany,
	}
}


NewMounter

func (plugin *portworxVolumePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
	return plugin.newMounterInternal(spec, pod.UID, plugin.util, plugin.host.GetMounter(plugin.GetPluginName()))
}


newMounterInternal

func (plugin *portworxVolumePlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager portworxManager, mounter mount.Interface) (volume.Mounter, error) {
	pwx, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}

	volumeID := pwx.VolumeID
	fsType := pwx.FSType

	return &portworxVolumeMounter{
		portworxVolume: &portworxVolume{
			podUID:          podUID,
			volName:         spec.Name(),
			volumeID:        volumeID,
			manager:         manager,
			mounter:         mounter,
			plugin:          plugin,
			MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, spec.Name(), plugin.host)),
		},
		fsType:      fsType,
		readOnly:    readOnly,
		diskMounter: util.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host)}, nil
}


NewUnmounter

func (plugin *portworxVolumePlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	return plugin.newUnmounterInternal(volName, podUID, plugin.util, plugin.host.GetMounter(plugin.GetPluginName()))
}


newUnmounterInternal

func (plugin *portworxVolumePlugin) newUnmounterInternal(volName string, podUID types.UID, manager portworxManager,
	mounter mount.Interface) (volume.Unmounter, error) {
	return &portworxVolumeUnmounter{
		&portworxVolume{
			podUID:          podUID,
			volName:         volName,
			manager:         manager,
			mounter:         mounter,
			plugin:          plugin,
			MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)),
		}}, nil
}


NewDeleter

func (plugin *portworxVolumePlugin) NewDeleter(logger klog.Logger, spec *volume.Spec) (volume.Deleter, error) {
	return plugin.newDeleterInternal(spec, plugin.util)
}


newDeleterInternal

func (plugin *portworxVolumePlugin) newDeleterInternal(spec *volume.Spec, manager portworxManager) (volume.Deleter, error) {
	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.PortworxVolume == nil {
		return nil, fmt.Errorf("spec.PersistentVolumeSource.PortworxVolume is nil")
	}

	return &portworxVolumeDeleter{
		portworxVolume: &portworxVolume{
			volName:  spec.Name(),
			volumeID: spec.PersistentVolume.Spec.PortworxVolume.VolumeID,
			manager:  manager,
			plugin:   plugin,
		}}, nil
}


NewProvisioner

func (plugin *portworxVolumePlugin) NewProvisioner(logger klog.Logger, options volume.VolumeOptions) (volume.Provisioner, error) {
	return plugin.newProvisionerInternal(options, plugin.util)
}


newProvisionerInternal

func (plugin *portworxVolumePlugin) newProvisionerInternal(options volume.VolumeOptions, manager portworxManager) (volume.Provisioner, error) {
	return &portworxVolumeProvisioner{
		portworxVolume: &portworxVolume{
			manager: manager,
			plugin:  plugin,
		},
		options: options,
	}, nil
}


RequiresFSResize

func (plugin *portworxVolumePlugin) RequiresFSResize() bool {
	return false
}


ExpandVolumeDevice

func (plugin *portworxVolumePlugin) ExpandVolumeDevice(
	spec *volume.Spec,
	newSize resource.Quantity,
	oldSize resource.Quantity) (resource.Quantity, error) {
	klog.V(4).Infof("Expanding: %s from %v to %v", spec.Name(), oldSize, newSize)
	err := plugin.util.ResizeVolume(spec, newSize, plugin.host)
	if err != nil {
		return oldSize, err
	}

	klog.V(4).Infof("Successfully resized %s to %v", spec.Name(), newSize)
	return newSize, nil
}


ConstructVolumeSpec

func (plugin *portworxVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
	portworxVolume := &v1.Volume{
		Name: volumeName,
		VolumeSource: v1.VolumeSource{
			PortworxVolume: &v1.PortworxVolumeSource{
				VolumeID: volumeName,
			},
		},
	}
	return volume.ReconstructedVolume{
		Spec: volume.NewSpecFromVolume(portworxVolume),
	}, nil
}


SupportsMountOption

func (plugin *portworxVolumePlugin) SupportsMountOption() bool {
	return false
}


SupportsBulkVolumeVerification

func (plugin *portworxVolumePlugin) SupportsBulkVolumeVerification() bool {
	return false
}


SupportsSELinuxContextMount

func (plugin *portworxVolumePlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return false, nil
}


getVolumeSource

func getVolumeSource(
	spec *volume.Spec) (*v1.PortworxVolumeSource, bool, error) {
	if spec.Volume != nil && spec.Volume.PortworxVolume != nil {
		return spec.Volume.PortworxVolume, spec.Volume.PortworxVolume.ReadOnly, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.PortworxVolume != nil {
		return spec.PersistentVolume.Spec.PortworxVolume, spec.ReadOnly, nil
	}

	return nil, false, fmt.Errorf("Spec does not reference a Portworx Volume type")
}


GetAttributes

func (b *portworxVolumeMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:       b.readOnly,
		Managed:        !b.readOnly,
		SELinuxRelabel: false,
	}
}


SetUp

// SetUp attaches the disk and bind mounts to the volume path.
func (b *portworxVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error {
	return b.SetUpAt(b.GetPath(), mounterArgs)
}


SetUpAt

// SetUpAt attaches the disk and bind mounts to the volume path.
func (b *portworxVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	notMnt, err := b.mounter.IsLikelyNotMountPoint(dir)
	klog.Infof("Portworx Volume set up. Dir: %s %v %v", dir, !notMnt, err)
	if err != nil && !os.IsNotExist(err) {
		klog.Errorf("Cannot validate mountpoint: %s", dir)
		return err
	}
	if !notMnt {
		return nil
	}

	attachOptions := make(map[string]string)
	attachOptions[attachContextKey] = dir
	attachOptions[attachHostKey] = b.plugin.host.GetHostName()
	if _, err := b.manager.AttachVolume(b, attachOptions); err != nil {
		return err
	}

	klog.V(4).Infof("Portworx Volume %s attached", b.volumeID)

	if err := os.MkdirAll(dir, 0750); err != nil {
		return err
	}

	if err := b.manager.MountVolume(b, dir); err != nil {
		return err
	}
	if !b.readOnly {
		volume.SetVolumeOwnership(b, dir, mounterArgs.FsGroup, mounterArgs.FSGroupChangePolicy, util.FSGroupCompleteHook(b.plugin, nil))
	}
	klog.Infof("Portworx Volume %s setup at %s", b.volumeID, dir)
	return nil
}


GetPath

func (pwx *portworxVolume) GetPath() string {
	return getPath(pwx.podUID, pwx.volName, pwx.plugin.host)
}


TearDown

// Unmounts the bind mount, and detaches the disk only if the PD
// resource was the last reference to that disk on the kubelet.
func (c *portworxVolumeUnmounter) TearDown() error {
	return c.TearDownAt(c.GetPath())
}


TearDownAt

// Unmounts the bind mount, and detaches the disk only if the PD
// resource was the last reference to that disk on the kubelet.
func (c *portworxVolumeUnmounter) TearDownAt(dir string) error {
	klog.Infof("Portworx Volume TearDown of %s", dir)

	if err := c.manager.UnmountVolume(c, dir); err != nil {
		return err
	}

	// Call Portworx Detach Volume.
	if err := c.manager.DetachVolume(c); err != nil {
		return err
	}

	return nil
}


GetPath

func (d *portworxVolumeDeleter) GetPath() string {
	return getPath(d.podUID, d.volName, d.plugin.host)
}


Delete

func (d *portworxVolumeDeleter) Delete() error {
	return d.manager.DeleteVolume(d)
}


Provision

func (c *portworxVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTopologies []v1.TopologySelectorTerm) (*v1.PersistentVolume, error) {
	if !util.ContainsAllAccessModes(c.plugin.GetAccessModes(), c.options.PVC.Spec.AccessModes) {
		return nil, fmt.Errorf("invalid AccessModes %v: only AccessModes %v are supported", c.options.PVC.Spec.AccessModes, c.plugin.GetAccessModes())
	}

	if util.CheckPersistentVolumeClaimModeBlock(c.options.PVC) {
		return nil, fmt.Errorf("%s does not support block volume provisioning", c.plugin.GetPluginName())
	}

	volumeID, sizeGiB, labels, err := c.manager.CreateVolume(c)
	if err != nil {
		return nil, err
	}

	pv := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name:   c.options.PVName,
			Labels: map[string]string{},
			Annotations: map[string]string{
				util.VolumeDynamicallyCreatedByKey: "portworx-volume-dynamic-provisioner",
			},
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeReclaimPolicy: c.options.PersistentVolumeReclaimPolicy,
			AccessModes:                   c.options.PVC.Spec.AccessModes,
			Capacity: v1.ResourceList{
				v1.ResourceName(v1.ResourceStorage): resource.MustParse(fmt.Sprintf("%dGi", sizeGiB)),
			},
			PersistentVolumeSource: v1.PersistentVolumeSource{
				PortworxVolume: &v1.PortworxVolumeSource{
					VolumeID: volumeID,
				},
			},
		},
	}

	if len(labels) != 0 {
		if pv.Labels == nil {
			pv.Labels = make(map[string]string)
		}
		for k, v := range labels {
			pv.Labels[k] = v
		}
	}

	if len(c.options.PVC.Spec.AccessModes) == 0 {
		pv.Spec.AccessModes = c.plugin.GetAccessModes()
	}

	return pv, nil
}


CreateVolume

// CreateVolume creates a Portworx volume.
func (util *portworxVolumeUtil) CreateVolume(p *portworxVolumeProvisioner) (string, int64, map[string]string, error) {
	driver, err := util.getPortworxDriver(p.plugin.host)
	if err != nil || driver == nil {
		klog.Errorf("Failed to get portworx driver. Err: %v", err)
		return "", 0, nil, err
	}

	klog.Infof("Creating Portworx volume for PVC: %v", p.options.PVC.Name)

	capacity := p.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	// Portworx Volumes are specified in GiB
	requestGiB, err := volumehelpers.RoundUpToGiB(capacity)
	if err != nil {
		return "", 0, nil, err
	}

	// Perform a best-effort parsing of parameters. Portworx 1.2.9 and later parses volume parameters from
	// spec.VolumeLabels. So even if below SpecFromOpts() fails to parse certain parameters or
	// doesn't support new parameters, the server-side processing will parse it correctly.
	// We still need to call SpecFromOpts() here to handle cases where someone is running Portworx 1.2.8 and lower.
	specHandler := osdspec.NewSpecHandler()
	spec, locator, source, _ := specHandler.SpecFromOpts(p.options.Parameters)
	if spec == nil {
		spec = specHandler.DefaultSpec()
	}

	// Pass all parameters as volume labels for Portworx server-side processing
	if spec.VolumeLabels == nil {
		spec.VolumeLabels = make(map[string]string, 0)
	}

	for k, v := range p.options.Parameters {
		spec.VolumeLabels[k] = v
	}

	// Update the requested size in the spec
	spec.Size = uint64(requestGiB * volumehelpers.GiB)

	// Change the Portworx Volume name to PV name
	if locator == nil {
		locator = &osdapi.VolumeLocator{
			VolumeLabels: make(map[string]string),
		}
	}
	locator.Name = p.options.PVName

	// Add claim Name as a part of Portworx Volume Labels
	locator.VolumeLabels[pvcClaimLabel] = p.options.PVC.Name
	locator.VolumeLabels[pvcNamespaceLabel] = p.options.PVC.Namespace

	for k, v := range p.options.PVC.Annotations {
		if _, present := spec.VolumeLabels[k]; present {
			klog.Warningf("not saving annotation: %s=%s in spec labels due to an existing key", k, v)
			continue
		}
		spec.VolumeLabels[k] = v
	}

	volumeID, err := driver.Create(locator, source, spec)
	if err != nil {
		klog.Errorf("Error creating Portworx Volume : %v", err)
		return "", 0, nil, err
	}

	klog.Infof("Successfully created Portworx volume for PVC: %v", p.options.PVC.Name)
	return volumeID, requestGiB, nil, err
}


DeleteVolume

// DeleteVolume deletes a Portworx volume
func (util *portworxVolumeUtil) DeleteVolume(d *portworxVolumeDeleter) error {
	driver, err := util.getPortworxDriver(d.plugin.host)
	if err != nil || driver == nil {
		klog.Errorf("Failed to get portworx driver. Err: %v", err)
		return err
	}

	err = driver.Delete(d.volumeID)
	if err != nil {
		klog.Errorf("Error deleting Portworx Volume (%v): %v", d.volName, err)
		return err
	}
	return nil
}


AttachVolume

// AttachVolume attaches a Portworx Volume
func (util *portworxVolumeUtil) AttachVolume(m *portworxVolumeMounter, attachOptions map[string]string) (string, error) {
	driver, err := util.getLocalPortworxDriver(m.plugin.host)
	if err != nil || driver == nil {
		klog.Errorf("Failed to get portworx driver. Err: %v", err)
		return "", err
	}

	devicePath, err := driver.Attach(m.volName, attachOptions)
	if err != nil {
		klog.Errorf("Error attaching Portworx Volume (%v): %v", m.volName, err)
		return "", err
	}
	return devicePath, nil
}


DetachVolume

// DetachVolume detaches a Portworx Volume
func (util *portworxVolumeUtil) DetachVolume(u *portworxVolumeUnmounter) error {
	driver, err := util.getLocalPortworxDriver(u.plugin.host)
	if err != nil || driver == nil {
		klog.Errorf("Failed to get portworx driver. Err: %v", err)
		return err
	}

	err = driver.Detach(u.volName, false /*doNotForceDetach*/)
	if err != nil {
		klog.Errorf("Error detaching Portworx Volume (%v): %v", u.volName, err)
		return err
	}
	return nil
}


MountVolume

// MountVolume mounts a Portworx Volume on the specified mountPath
func (util *portworxVolumeUtil) MountVolume(m *portworxVolumeMounter, mountPath string) error {
	driver, err := util.getLocalPortworxDriver(m.plugin.host)
	if err != nil || driver == nil {
		klog.Errorf("Failed to get portworx driver. Err: %v", err)
		return err
	}

	err = driver.Mount(m.volName, mountPath)
	if err != nil {
		klog.Errorf("Error mounting Portworx Volume (%v) on Path (%v): %v", m.volName, mountPath, err)
		return err
	}
	return nil
}


UnmountVolume

// UnmountVolume unmounts a Portworx Volume
func (util *portworxVolumeUtil) UnmountVolume(u *portworxVolumeUnmounter, mountPath string) error {
	driver, err := util.getLocalPortworxDriver(u.plugin.host)
	if err != nil || driver == nil {
		klog.Errorf("Failed to get portworx driver. Err: %v", err)
		return err
	}

	err = driver.Unmount(u.volName, mountPath)
	if err != nil {
		klog.Errorf("Error unmounting Portworx Volume (%v) on Path (%v): %v", u.volName, mountPath, err)
		return err
	}
	return nil
}


ResizeVolume

func (util *portworxVolumeUtil) ResizeVolume(spec *volume.Spec, newSize resource.Quantity, volumeHost volume.VolumeHost) error {
	driver, err := util.getPortworxDriver(volumeHost)
	if err != nil || driver == nil {
		klog.Errorf("Failed to get portworx driver. Err: %v", err)
		return err
	}

	vols, err := driver.Inspect([]string{spec.Name()})
	if err != nil {
		return err
	}

	if len(vols) != 1 {
		return fmt.Errorf("failed to inspect Portworx volume: %s. Found: %d volumes", spec.Name(), len(vols))
	}

	vol := vols[0]
	tBytes, err := volumehelpers.RoundUpToB(newSize)
	if err != nil {
		return err
	}
	newSizeInBytes := uint64(tBytes)
	if vol.Spec.Size >= newSizeInBytes {
		klog.Infof("Portworx volume: %s already at size: %d greater than or equal to new "+
			"requested size: %d. Skipping resize.", spec.Name(), vol.Spec.Size, newSizeInBytes)
		return nil
	}

	vol.Spec.Size = newSizeInBytes
	err = driver.Set(spec.Name(), vol.Locator, vol.Spec)
	if err != nil {
		return err
	}

	// check if the volume's size actually got updated
	vols, err = driver.Inspect([]string{spec.Name()})
	if err != nil {
		return err
	}

	if len(vols) != 1 {
		return fmt.Errorf("failed to inspect resized Portworx volume: %s. Found: %d volumes", spec.Name(), len(vols))
	}

	updatedVol := vols[0]
	if updatedVol.Spec.Size < vol.Spec.Size {
		return fmt.Errorf("Portworx volume: %s doesn't match expected size after resize. expected:%v actual:%v",
			spec.Name(), vol.Spec.Size, updatedVol.Spec.Size)
	}

	return nil
}


isClientValid

func isClientValid(client *osdclient.Client) (bool, error) {
	if client == nil {
		return false, nil
	}

	_, err := client.Versions(osdapi.OsdVolumePath)
	if err != nil {
		klog.Errorf("portworx client failed driver versions check. Err: %v", err)
		return false, err
	}

	return true, nil
}


createDriverClient

func createDriverClient(hostname string, port int32) (*osdclient.Client, error) {
	client, err := volumeclient.NewDriverClient(fmt.Sprintf("http://%s", net.JoinHostPort(hostname, strconv.Itoa(int(port)))),
		pxdDriverName, osdDriverVersion, pxDriverName)
	if err != nil {
		return nil, err
	}

	isValid, err := isClientValid(client)
	if isValid {
		return client, nil
	}
	return nil, err
}


getPortworxDriver

// getPortworxDriver returns a Portworx volume driver which can be used for cluster wide operations.
//
//	Operations like create and delete volume don't need to be restricted to local volume host since
//	any node in the Portworx cluster can coordinate the create/delete request and forward the operations to
//	the Portworx node that will own/owns the data.
func (util *portworxVolumeUtil) getPortworxDriver(volumeHost volume.VolumeHost) (volumeapi.VolumeDriver, error) {
	// check if existing saved client is valid
	if isValid, _ := isClientValid(util.portworxClient); isValid {
		return volumeclient.VolumeDriver(util.portworxClient), nil
	}

	// create new client
	var err error
	util.portworxClient, err = createDriverClient(volumeHost.GetHostName(), osdMgmtDefaultPort) // for backward compatibility
	if err != nil || util.portworxClient == nil {
		// Create client from portworx k8s service.
		svc, err := getPortworxService(volumeHost)
		if err != nil {
			return nil, err
		}

		// The port here is always the default one since  it's the service port
		util.portworxClient, err = createDriverClient(svc.Spec.ClusterIP, osdMgmtDefaultPort)
		if err != nil || util.portworxClient == nil {
			klog.Errorf("Failed to connect to portworx service. Err: %v", err)
			return nil, err
		}

		klog.Infof("Using portworx cluster service at: %v:%d as api endpoint",
			svc.Spec.ClusterIP, osdMgmtDefaultPort)
	} else {
		klog.Infof("Using portworx service at: %v:%d as api endpoint",
			volumeHost.GetHostName(), osdMgmtDefaultPort)
	}

	return volumeclient.VolumeDriver(util.portworxClient), nil
}


getLocalPortworxDriver

// getLocalPortworxDriver returns driver connected to Portworx API server on volume host.
//
//	This is required to force certain operations (mount, unmount, detach, attach) to
//	go to the volume host instead of the k8s service which might route it to any host. This pertains to how
//	Portworx mounts and attaches a volume to the running container. The node getting these requests needs to
//	see the pod container mounts (specifically /var/lib/kubelet/pods/<pod_id>)
func (util *portworxVolumeUtil) getLocalPortworxDriver(volumeHost volume.VolumeHost) (volumeapi.VolumeDriver, error) {
	if util.portworxClient != nil {
		// check if existing saved client is valid
		if isValid, _ := isClientValid(util.portworxClient); isValid {
			return volumeclient.VolumeDriver(util.portworxClient), nil
		}
	}

	// Lookup port
	svc, err := getPortworxService(volumeHost)
	if err != nil {
		return nil, err
	}

	osgMgmtPort := lookupPXAPIPortFromService(svc)
	util.portworxClient, err = createDriverClient(volumeHost.GetHostName(), osgMgmtPort)
	if err != nil {
		return nil, err
	}

	klog.Infof("Using portworx local service at: %v:%d as api endpoint",
		volumeHost.GetHostName(), osgMgmtPort)
	return volumeclient.VolumeDriver(util.portworxClient), nil
}


lookupPXAPIPortFromService

// lookupPXAPIPortFromService goes over all the ports in the given service and returns the target
// port for osdMgmtDefaultPort
func lookupPXAPIPortFromService(svc *v1.Service) int32 {
	for _, p := range svc.Spec.Ports {
		if p.Port == osdMgmtDefaultPort {
			return p.TargetPort.IntVal
		}
	}
	return osdMgmtDefaultPort // default
}


getPortworxService

// getPortworxService returns the portworx cluster service from the API server
func getPortworxService(host volume.VolumeHost) (*v1.Service, error) {
	kubeClient := host.GetKubeClient()
	if kubeClient == nil {
		err := fmt.Errorf("failed to get kubeclient when creating portworx client")
		klog.Errorf(err.Error())
		return nil, err
	}

	opts := metav1.GetOptions{}
	svc, err := kubeClient.CoreV1().Services(api.NamespaceSystem).Get(context.TODO(), pxServiceName, opts)
	if err != nil {
		klog.Errorf("Failed to get service. Err: %v", err)
		return nil, err
	}

	if svc == nil {
		err = fmt.Errorf("service: %v not found. Consult Portworx docs to deploy it", pxServiceName)
		klog.Errorf(err.Error())
		return nil, err
	}

	return svc, nil
}


ProbeVolumePlugins

// ProbeVolumePlugins is the entry point for plugin detection in a package.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&projectedPlugin{}}
}


wrappedVolumeSpec

func wrappedVolumeSpec() volume.Spec {
	return volume.Spec{
		Volume: &v1.Volume{
			VolumeSource: v1.VolumeSource{
				EmptyDir: &v1.EmptyDirVolumeSource{Medium: v1.StorageMediumMemory},
			},
		},
	}
}


getPath

func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
	return host.GetPodVolumeDir(uid, utilstrings.EscapeQualifiedName(projectedPluginName), volName)
}


Init

func (plugin *projectedPlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	plugin.getSecret = host.GetSecretFunc()
	plugin.getConfigMap = host.GetConfigMapFunc()
	plugin.getServiceAccountToken = host.GetServiceAccountTokenFunc()
	plugin.deleteServiceAccountToken = host.DeleteServiceAccountTokenFunc()
	return nil
}


GetPluginName

func (plugin *projectedPlugin) GetPluginName() string {
	return projectedPluginName
}


GetVolumeName

func (plugin *projectedPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	_, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return spec.Name(), nil
}


CanSupport

func (plugin *projectedPlugin) CanSupport(spec *volume.Spec) bool {
	return spec.Volume != nil && spec.Volume.Projected != nil
}


RequiresRemount

func (plugin *projectedPlugin) RequiresRemount(spec *volume.Spec) bool {
	return true
}


SupportsMountOption

func (plugin *projectedPlugin) SupportsMountOption() bool {
	return false
}


SupportsBulkVolumeVerification

func (plugin *projectedPlugin) SupportsBulkVolumeVerification() bool {
	return false
}


SupportsSELinuxContextMount

func (plugin *projectedPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return false, nil
}


NewMounter

func (plugin *projectedPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) {
	return &projectedVolumeMounter{
		projectedVolume: &projectedVolume{
			volName:         spec.Name(),
			sources:         spec.Volume.Projected.Sources,
			podUID:          pod.UID,
			plugin:          plugin,
			MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsDu(getPath(pod.UID, spec.Name(), plugin.host))),
		},
		source: *spec.Volume.Projected,
		pod:    pod,
		opts:   &opts,
	}, nil
}


NewUnmounter

func (plugin *projectedPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	return &projectedVolumeUnmounter{
		&projectedVolume{
			volName:         volName,
			podUID:          podUID,
			plugin:          plugin,
			MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsDu(getPath(podUID, volName, plugin.host))),
		},
	}, nil
}


ConstructVolumeSpec

func (plugin *projectedPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
	projectedVolume := &v1.Volume{
		Name: volumeName,
		VolumeSource: v1.VolumeSource{
			Projected: &v1.ProjectedVolumeSource{},
		},
	}

	return volume.ReconstructedVolume{
		Spec: volume.NewSpecFromVolume(projectedVolume),
	}, nil
}


GetPath

func (sv *projectedVolume) GetPath() string {
	return getPath(sv.podUID, sv.volName, sv.plugin.host)
}


GetAttributes

func (sv *projectedVolume) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:       true,
		Managed:        true,
		SELinuxRelabel: true,
	}

}


SetUp

func (s *projectedVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error {
	return s.SetUpAt(s.GetPath(), mounterArgs)
}


SetUpAt

func (s *projectedVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	klog.V(3).Infof("Setting up volume %v for pod %v at %v", s.volName, s.pod.UID, dir)

	wrapped, err := s.plugin.host.NewWrapperMounter(s.volName, wrappedVolumeSpec(), s.pod, *s.opts)
	if err != nil {
		return err
	}

	data, err := s.collectData(mounterArgs)
	if err != nil {
		klog.Errorf("Error preparing data for projected volume %v for pod %v/%v: %s", s.volName, s.pod.Namespace, s.pod.Name, err.Error())
		return err
	}

	setupSuccess := false
	if err := wrapped.SetUpAt(dir, mounterArgs); err != nil {
		return err
	}

	if err := volumeutil.MakeNestedMountpoints(s.volName, dir, *s.pod); err != nil {
		return err
	}

	defer func() {
		// Clean up directories if setup fails
		if !setupSuccess {
			unmounter, unmountCreateErr := s.plugin.NewUnmounter(s.volName, s.podUID)
			if unmountCreateErr != nil {
				klog.Errorf("error cleaning up mount %s after failure. Create unmounter failed with %v", s.volName, unmountCreateErr)
				return
			}
			tearDownErr := unmounter.TearDown()
			if tearDownErr != nil {
				klog.Errorf("error tearing down volume %s with : %v", s.volName, tearDownErr)
			}
		}
	}()

	writerContext := fmt.Sprintf("pod %v/%v volume %v", s.pod.Namespace, s.pod.Name, s.volName)
	writer, err := volumeutil.NewAtomicWriter(dir, writerContext)
	if err != nil {
		klog.Errorf("Error creating atomic writer: %v", err)
		return err
	}

	setPerms := func(_ string) error {
		// This may be the first time writing and new files get created outside the timestamp subdirectory:
		// change the permissions on the whole volume and not only in the timestamp directory.
		return volume.SetVolumeOwnership(s, dir, mounterArgs.FsGroup, nil /*fsGroupChangePolicy*/, volumeutil.FSGroupCompleteHook(s.plugin, nil))
	}
	err = writer.Write(data, setPerms)
	if err != nil {
		klog.Errorf("Error writing payload to dir: %v", err)
		return err
	}

	setupSuccess = true
	return nil
}


collectData

func (s *projectedVolumeMounter) collectData(mounterArgs volume.MounterArgs) (map[string]volumeutil.FileProjection, error) {
	if s.source.DefaultMode == nil {
		return nil, fmt.Errorf("no defaultMode used, not even the default value for it")
	}

	kubeClient := s.plugin.host.GetKubeClient()
	if kubeClient == nil {
		return nil, fmt.Errorf("cannot setup projected volume %v because kube client is not configured", s.volName)
	}

	errlist := []error{}
	payload := make(map[string]volumeutil.FileProjection)
	for _, source := range s.source.Sources {
		switch {
		case source.Secret != nil:
			optional := source.Secret.Optional != nil && *source.Secret.Optional
			secretapi, err := s.plugin.getSecret(s.pod.Namespace, source.Secret.Name)
			if err != nil {
				if !(errors.IsNotFound(err) && optional) {
					klog.Errorf("Couldn't get secret %v/%v: %v", s.pod.Namespace, source.Secret.Name, err)
					errlist = append(errlist, err)
					continue
				}
				secretapi = &v1.Secret{
					ObjectMeta: metav1.ObjectMeta{
						Namespace: s.pod.Namespace,
						Name:      source.Secret.Name,
					},
				}
			}
			secretPayload, err := secret.MakePayload(source.Secret.Items, secretapi, s.source.DefaultMode, optional)
			if err != nil {
				klog.Errorf("Couldn't get secret payload %v/%v: %v", s.pod.Namespace, source.Secret.Name, err)
				errlist = append(errlist, err)
				continue
			}
			for k, v := range secretPayload {
				payload[k] = v
			}
		case source.ConfigMap != nil:
			optional := source.ConfigMap.Optional != nil && *source.ConfigMap.Optional
			configMap, err := s.plugin.getConfigMap(s.pod.Namespace, source.ConfigMap.Name)
			if err != nil {
				if !(errors.IsNotFound(err) && optional) {
					klog.Errorf("Couldn't get configMap %v/%v: %v", s.pod.Namespace, source.ConfigMap.Name, err)
					errlist = append(errlist, err)
					continue
				}
				configMap = &v1.ConfigMap{
					ObjectMeta: metav1.ObjectMeta{
						Namespace: s.pod.Namespace,
						Name:      source.ConfigMap.Name,
					},
				}
			}
			configMapPayload, err := configmap.MakePayload(source.ConfigMap.Items, configMap, s.source.DefaultMode, optional)
			if err != nil {
				klog.Errorf("Couldn't get configMap payload %v/%v: %v", s.pod.Namespace, source.ConfigMap.Name, err)
				errlist = append(errlist, err)
				continue
			}
			for k, v := range configMapPayload {
				payload[k] = v
			}
		case source.DownwardAPI != nil:
			downwardAPIPayload, err := downwardapi.CollectData(source.DownwardAPI.Items, s.pod, s.plugin.host, s.source.DefaultMode)
			if err != nil {
				errlist = append(errlist, err)
				continue
			}
			for k, v := range downwardAPIPayload {
				payload[k] = v
			}
		case source.ServiceAccountToken != nil:
			tp := source.ServiceAccountToken

			// When FsGroup is set, we depend on SetVolumeOwnership to
			// change from 0600 to 0640.
			mode := *s.source.DefaultMode
			if mounterArgs.FsUser != nil || mounterArgs.FsGroup != nil {
				mode = 0600
			}

			var auds []string
			if len(tp.Audience) != 0 {
				auds = []string{tp.Audience}
			}
			tr, err := s.plugin.getServiceAccountToken(s.pod.Namespace, s.pod.Spec.ServiceAccountName, &authenticationv1.TokenRequest{
				Spec: authenticationv1.TokenRequestSpec{
					Audiences:         auds,
					ExpirationSeconds: tp.ExpirationSeconds,
					BoundObjectRef: &authenticationv1.BoundObjectReference{
						APIVersion: "v1",
						Kind:       "Pod",
						Name:       s.pod.Name,
						UID:        s.pod.UID,
					},
				},
			})
			if err != nil {
				errlist = append(errlist, err)
				continue
			}
			payload[tp.Path] = volumeutil.FileProjection{
				Data:   []byte(tr.Status.Token),
				Mode:   mode,
				FsUser: mounterArgs.FsUser,
			}
		}
	}
	return payload, utilerrors.NewAggregate(errlist)
}


TearDown

func (c *projectedVolumeUnmounter) TearDown() error {
	return c.TearDownAt(c.GetPath())
}


TearDownAt

func (c *projectedVolumeUnmounter) TearDownAt(dir string) error {
	klog.V(3).Infof("Tearing down volume %v for pod %v at %v", c.volName, c.podUID, dir)

	wrapped, err := c.plugin.host.NewWrapperUnmounter(c.volName, wrappedVolumeSpec(), c.podUID)
	if err != nil {
		return err
	}
	if err = wrapped.TearDownAt(dir); err != nil {
		return err
	}

	c.plugin.deleteServiceAccountToken(c.podUID)
	return nil
}


getVolumeSource

func getVolumeSource(spec *volume.Spec) (*v1.ProjectedVolumeSource, bool, error) {
	if spec.Volume != nil && spec.Volume.Projected != nil {
		return spec.Volume.Projected, spec.ReadOnly, nil
	}

	return nil, false, fmt.Errorf("Spec does not reference a projected volume type")
}


NewAttacher

// NewAttacher implements AttachableVolumePlugin.NewAttacher.
func (plugin *rbdPlugin) NewAttacher() (volume.Attacher, error) {
	return plugin.newAttacherInternal(&rbdUtil{})
}


NewDeviceMounter

// NewDeviceMounter implements DeviceMountableVolumePlugin.NewDeviceMounter
func (plugin *rbdPlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
	return plugin.NewAttacher()
}


newAttacherInternal

func (plugin *rbdPlugin) newAttacherInternal(manager diskManager) (volume.Attacher, error) {
	return &rbdAttacher{
		plugin:  plugin,
		manager: manager,
		mounter: volutil.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host),
	}, nil
}


NewDetacher

// NewDetacher implements AttachableVolumePlugin.NewDetacher.
func (plugin *rbdPlugin) NewDetacher() (volume.Detacher, error) {
	return plugin.newDetacherInternal(&rbdUtil{})
}


NewDeviceUnmounter

// NewDeviceUnmounter implements DeviceMountableVolumePlugin.NewDeviceUnmounter
func (plugin *rbdPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
	return plugin.NewDetacher()
}


newDetacherInternal

func (plugin *rbdPlugin) newDetacherInternal(manager diskManager) (volume.Detacher, error) {
	return &rbdDetacher{
		plugin:  plugin,
		manager: manager,
		mounter: volutil.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host),
	}, nil
}


GetDeviceMountRefs

// GetDeviceMountRefs implements AttachableVolumePlugin.GetDeviceMountRefs.
func (plugin *rbdPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
	mounter := plugin.host.GetMounter(plugin.GetPluginName())
	return mounter.GetMountRefs(deviceMountPath)
}


CanAttach

func (plugin *rbdPlugin) CanAttach(spec *volume.Spec) (bool, error) {
	return true, nil
}


CanDeviceMount

func (plugin *rbdPlugin) CanDeviceMount(spec *volume.Spec) (bool, error) {
	return true, nil
}


Attach

// Attach implements Attacher.Attach.
// We do not lock image here, because it requires kube-controller-manager to
// access external `rbd` utility. And there is no need since AttachDetach
// controller will not try to attach RWO volumes which are already attached to
// other nodes.
func (attacher *rbdAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) {
	return "", nil
}


VolumesAreAttached

// VolumesAreAttached implements Attacher.VolumesAreAttached.
// There is no way to confirm whether the volume is attached or not from
// outside of the kubelet node. This method needs to return true always, like
// iSCSI, FC plugin.
func (attacher *rbdAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.NodeName) (map[*volume.Spec]bool, error) {
	volumesAttachedCheck := make(map[*volume.Spec]bool)
	for _, spec := range specs {
		volumesAttachedCheck[spec] = true
	}
	return volumesAttachedCheck, nil
}


WaitForAttach

// WaitForAttach implements Attacher.WaitForAttach. It's called by kubelet to
// attach volume onto the node.
// This method is idempotent, callers are responsible for retrying on failure.
func (attacher *rbdAttacher) WaitForAttach(spec *volume.Spec, devicePath string, pod *v1.Pod, timeout time.Duration) (string, error) {
	klog.V(4).Infof("rbd: waiting for attach volume (name: %s) for pod (name: %s, uid: %s)", spec.Name(), pod.Name, pod.UID)
	mounter, err := attacher.plugin.createMounterFromVolumeSpecAndPod(spec, pod)
	if err != nil {
		klog.Warningf("failed to create mounter: %v", spec)
		return "", err
	}
	realDevicePath, err := attacher.manager.AttachDisk(*mounter)
	if err != nil {
		return "", err
	}
	klog.V(3).Infof("rbd: successfully wait for attach volume (spec: %s, pool: %s, image: %s) at %s", spec.Name(), mounter.Pool, mounter.Image, realDevicePath)
	return realDevicePath, nil
}


GetDeviceMountPath

// GetDeviceMountPath implements Attacher.GetDeviceMountPath.
func (attacher *rbdAttacher) GetDeviceMountPath(spec *volume.Spec) (string, error) {
	img, err := getVolumeSourceImage(spec)
	if err != nil {
		return "", err
	}
	pool, err := getVolumeSourcePool(spec)
	if err != nil {
		return "", err
	}
	return makePDNameInternal(attacher.plugin.host, pool, img), nil
}


MountDevice

// MountDevice implements Attacher.MountDevice. It is called by the kubelet to
// mount device at the given mount path.
// This method is idempotent, callers are responsible for retrying on failure.
func (attacher *rbdAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string, mountArgs volume.DeviceMounterArgs) error {
	klog.V(4).Infof("rbd: mouting device %s to %s", devicePath, deviceMountPath)
	notMnt, err := attacher.mounter.IsLikelyNotMountPoint(deviceMountPath)
	if err != nil {
		if os.IsNotExist(err) {
			if err := os.MkdirAll(deviceMountPath, 0750); err != nil {
				return err
			}
			notMnt = true
		} else {
			return err
		}
	}
	if !notMnt {
		return nil
	}
	fstype, err := getVolumeSourceFSType(spec)
	if err != nil {
		return err
	}
	ro, err := getVolumeSourceReadOnly(spec)
	if err != nil {
		return err
	}
	options := []string{}
	if ro {
		options = append(options, "ro")
	}
	if mountArgs.SELinuxLabel != "" {
		options = volutil.AddSELinuxMountOption(options, mountArgs.SELinuxLabel)
	}
	mountOptions := volutil.MountOptionFromSpec(spec, options...)

	err = attacher.mounter.FormatAndMount(devicePath, deviceMountPath, fstype, mountOptions)
	if err != nil {
		os.Remove(deviceMountPath)
		return fmt.Errorf("rbd: failed to mount device %s at %s (fstype: %s), error %v", devicePath, deviceMountPath, fstype, err)
	}
	klog.V(3).Infof("rbd: successfully mount device %s at %s (fstype: %s)", devicePath, deviceMountPath, fstype)
	return nil
}


UnmountDevice

// UnmountDevice implements Detacher.UnmountDevice. It unmounts the global
// mount of the RBD image. This is called once all bind mounts have been
// unmounted.
// Internally, it does four things:
//   - Unmount device from deviceMountPath.
//   - Detach device from the node.
//   - Remove lock if found. (No need to check volume readonly or not, because
//     device is not on the node anymore, it's safe to remove lock.)
//   - Remove the deviceMountPath at last.
//
// This method is idempotent, callers are responsible for retrying on failure.
func (detacher *rbdDetacher) UnmountDevice(deviceMountPath string) error {
	if pathExists, pathErr := mount.PathExists(deviceMountPath); pathErr != nil {
		return fmt.Errorf("error checking if path exists: %v", pathErr)
	} else if !pathExists {
		klog.Warningf("Warning: Unmount skipped because path does not exist: %v", deviceMountPath)
		return nil
	}
	devicePath, _, err := mount.GetDeviceNameFromMount(detacher.mounter, deviceMountPath)
	if err != nil {
		return err
	}
	// Unmount the device from the device mount point.
	notMnt, err := detacher.mounter.IsLikelyNotMountPoint(deviceMountPath)
	if err != nil {
		return err
	}
	if !notMnt {
		klog.V(4).Infof("rbd: unmouting device mountpoint %s", deviceMountPath)
		if err = detacher.mounter.Unmount(deviceMountPath); err != nil {
			return err
		}
		klog.V(3).Infof("rbd: successfully unmount device mountpath %s", deviceMountPath)
	}

	// Get devicePath from deviceMountPath if devicePath is empty
	if devicePath == "" {
		rbdImageInfo, err := getRbdImageInfo(deviceMountPath)
		if err != nil {
			return err
		}
		found := false
		devicePath, found = getRbdDevFromImageAndPool(rbdImageInfo.pool, rbdImageInfo.name)
		if !found {
			klog.Warningf("rbd: can't found devicePath for %v. Device is already unmounted, Image %v, Pool %v", deviceMountPath, rbdImageInfo.pool, rbdImageInfo.name)
		}
	}

	if devicePath != "" {
		klog.V(4).Infof("rbd: detaching device %s", devicePath)
		err = detacher.manager.DetachDisk(detacher.plugin, deviceMountPath, devicePath)
		if err != nil {
			return err
		}
		klog.V(3).Infof("rbd: successfully detach device %s", devicePath)
	}
	err = os.Remove(deviceMountPath)
	if err != nil {
		return err
	}
	klog.V(3).Infof("rbd: successfully remove device mount point %s", deviceMountPath)
	return nil
}


Detach

// Detach implements Detacher.Detach.
func (detacher *rbdDetacher) Detach(volumeName string, nodeName types.NodeName) error {
	return nil
}


diskSetUp

// utility to mount a disk based filesystem
func diskSetUp(manager diskManager, b rbdMounter, volPath string, mounter mount.Interface, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy) error {
	globalPDPath := manager.MakeGlobalPDName(*b.rbd)
	notMnt, err := mounter.IsLikelyNotMountPoint(globalPDPath)
	if err != nil && !os.IsNotExist(err) {
		klog.Errorf("cannot validate mountpoint: %s", globalPDPath)
		return err
	}
	if notMnt {
		return fmt.Errorf("no device is mounted at %s", globalPDPath)
	}

	notMnt, err = mounter.IsLikelyNotMountPoint(volPath)
	if err != nil && !os.IsNotExist(err) {
		klog.Errorf("cannot validate mountpoint: %s", volPath)
		return err
	}
	if !notMnt {
		return nil
	}

	if err := os.MkdirAll(volPath, 0750); err != nil {
		klog.Errorf("failed to mkdir:%s", volPath)
		return err
	}
	// Perform a bind mount to the full path to allow duplicate mounts of the same disk.
	options := []string{"bind"}
	if (&b).GetAttributes().ReadOnly {
		options = append(options, "ro")
	}
	mountOptions := util.JoinMountOptions(b.mountOptions, options)
	err = mounter.Mount(globalPDPath, volPath, "", mountOptions)
	if err != nil {
		klog.Errorf("failed to bind mount:%s", globalPDPath)
		return err
	}
	klog.V(3).Infof("rbd: successfully bind mount %s to %s with options %v", globalPDPath, volPath, mountOptions)

	if !b.ReadOnly {
		volume.SetVolumeOwnership(&b, volPath, fsGroup, fsGroupChangePolicy, util.FSGroupCompleteHook(b.plugin, nil))
	}

	return nil
}


diskTearDown

// utility to tear down a disk based filesystem
func diskTearDown(manager diskManager, c rbdUnmounter, volPath string, mounter mount.Interface) error {
	notMnt, err := mounter.IsLikelyNotMountPoint(volPath)
	if err != nil && !os.IsNotExist(err) {
		klog.Errorf("cannot validate mountpoint: %s", volPath)
		return err
	}
	if notMnt {
		klog.V(3).Infof("volume path %s is not a mountpoint, deleting", volPath)
		return os.Remove(volPath)
	}

	// Unmount the bind-mount inside this pod.
	if err := mounter.Unmount(volPath); err != nil {
		klog.Errorf("failed to unmount %s", volPath)
		return err
	}

	notMnt, mntErr := mounter.IsLikelyNotMountPoint(volPath)
	if mntErr != nil && !os.IsNotExist(mntErr) {
		klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
		return mntErr
	}
	if notMnt {
		if err := os.Remove(volPath); err != nil {
			klog.V(2).Infof("Error removing mountpoint %s: %v", volPath, err)
			return err
		}
	}
	return nil
}


ProbeVolumePlugins

// ProbeVolumePlugins is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&rbdPlugin{}}
}


getPath

func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
	return host.GetPodVolumeDir(uid, utilstrings.EscapeQualifiedName(rbdPluginName), volName)
}


IsMigratedToCSI

func (plugin *rbdPlugin) IsMigratedToCSI() bool {
	return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationRBD)
}


Init

func (plugin *rbdPlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	return nil
}


GetPluginName

func (plugin *rbdPlugin) GetPluginName() string {
	return rbdPluginName
}


GetVolumeName

func (plugin *rbdPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	pool, err := getVolumeSourcePool(spec)
	if err != nil {
		return "", err
	}
	img, err := getVolumeSourceImage(spec)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf(
		"%v:%v",
		pool,
		img), nil
}


CanSupport

func (plugin *rbdPlugin) CanSupport(spec *volume.Spec) bool {
	return (spec.Volume != nil && spec.Volume.RBD != nil) || (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD != nil)
}


RequiresRemount

func (plugin *rbdPlugin) RequiresRemount(spec *volume.Spec) bool {
	return false
}


SupportsMountOption

func (plugin *rbdPlugin) SupportsMountOption() bool {
	return true
}


SupportsBulkVolumeVerification

func (plugin *rbdPlugin) SupportsBulkVolumeVerification() bool {
	return false
}


SupportsSELinuxContextMount

func (plugin *rbdPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return true, nil
}


GetAccessModes

func (plugin *rbdPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
	return []v1.PersistentVolumeAccessMode{
		v1.ReadWriteOnce,
		v1.ReadOnlyMany,
	}
}


getAdminAndSecret

func (plugin *rbdPlugin) getAdminAndSecret(spec *volume.Spec) (string, string, error) {
	class, err := volutil.GetClassForVolume(plugin.host.GetKubeClient(), spec.PersistentVolume)
	if err != nil {
		return "", "", err
	}
	adminSecretName := ""
	adminSecretNamespace := rbdDefaultAdminSecretNamespace
	admin := ""

	for k, v := range class.Parameters {
		switch dstrings.ToLower(k) {
		case "adminid":
			admin = v
		case "adminsecretname":
			adminSecretName = v
		case "adminsecretnamespace":
			adminSecretNamespace = v
		}
	}

	if admin == "" {
		admin = rbdDefaultAdminID
	}
	secret, err := parsePVSecret(adminSecretNamespace, adminSecretName, plugin.host.GetKubeClient())
	if err != nil {
		return admin, "", fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err)
	}

	return admin, secret, nil
}


ExpandVolumeDevice

func (plugin *rbdPlugin) ExpandVolumeDevice(spec *volume.Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) {
	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD == nil {
		return oldSize, fmt.Errorf("spec.PersistentVolume.Spec.RBD is nil")
	}

	// get admin and secret
	admin, secret, err := plugin.getAdminAndSecret(spec)
	if err != nil {
		return oldSize, err
	}

	expander := &rbdVolumeExpander{
		rbdMounter: &rbdMounter{
			rbd: &rbd{
				volName: spec.Name(),
				Image:   spec.PersistentVolume.Spec.RBD.RBDImage,
				Pool:    spec.PersistentVolume.Spec.RBD.RBDPool,
				plugin:  plugin,
				manager: &rbdUtil{},
				mounter: &mount.SafeFormatAndMount{Interface: plugin.host.GetMounter(plugin.GetPluginName())},
				exec:    plugin.host.GetExec(plugin.GetPluginName()),
			},
			Mon:         spec.PersistentVolume.Spec.RBD.CephMonitors,
			adminID:     admin,
			adminSecret: secret,
		},
	}

	expandedSize, err := expander.ResizeImage(oldSize, newSize)
	if err != nil {
		return oldSize, err
	}
	return expandedSize, nil

}


NodeExpand

func (plugin *rbdPlugin) NodeExpand(resizeOptions volume.NodeResizeOptions) (bool, error) {
	fsVolume, err := volutil.CheckVolumeModeFilesystem(resizeOptions.VolumeSpec)
	if err != nil {
		return false, fmt.Errorf("error checking VolumeMode: %v", err)
	}
	// if volume is not a fs file system, there is nothing for us to do here.
	if !fsVolume {
		return true, nil
	}
	_, err = volutil.GenericResizeFS(plugin.host, plugin.GetPluginName(), resizeOptions.DevicePath, resizeOptions.DeviceMountPath)
	if err != nil {
		return false, err
	}
	return true, nil
}


ResizeImage

func (expander *rbdVolumeExpander) ResizeImage(oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) {
	return expander.manager.ExpandImage(expander, oldSize, newSize)
}


RequiresFSResize

func (plugin *rbdPlugin) RequiresFSResize() bool {
	return true
}