func (plugin *flexVolumePlugin) getDeviceMountPath(spec *volume.Spec) (string, error) {
	volumeName, err := plugin.GetVolumeName(spec)
	if err != nil {
		return "", fmt.Errorf("GetVolumeName failed from getDeviceMountPath: %s", err)
	}

	mountsDir := filepath.Join(plugin.host.GetPluginDir(flexVolumePluginName), plugin.driverName, "mounts")
	return filepath.Join(mountsDir, volumeName), nil
}


func (plugin *flexVolumePlugin) RequiresFSResize() bool {
	return plugin.capabilities.RequiresFSResize
}


// GetDynamicPluginProber creates dynamic plugin prober
func GetDynamicPluginProber(pluginDir string, runner exec.Interface) volume.DynamicPluginProber {
	return &flexVolumeProber{
		pluginDir: pluginDir,
		watcher:   utilfs.NewFsnotifyWatcher(),
		factory:   pluginFactory{},
		runner:    runner,
		fs:        &utilfs.DefaultFs{},
	}
}


func (prober *flexVolumeProber) Init() error {
	prober.testAndSetProbeAllNeeded(true)
	prober.eventsMap = map[string]volume.ProbeOperation{}

	if err := prober.createPluginDir(); err != nil {
		return err
	}
	if err := prober.initWatcher(); err != nil {
		return err
	}

	return nil
}


// If probeAllNeeded is true, probe all pluginDir
// else probe events in eventsMap
func (prober *flexVolumeProber) Probe() (events []volume.ProbeEvent, err error) {
	if prober.probeAllNeeded {
		prober.testAndSetProbeAllNeeded(false)
		return prober.probeAll()
	}

	return prober.probeMap()
}


func (prober *flexVolumeProber) probeMap() (events []volume.ProbeEvent, err error) {
	// TODO use a concurrent map to avoid Locking the entire map
	prober.mutex.Lock()
	defer prober.mutex.Unlock()
	probeEvents := []volume.ProbeEvent{}
	allErrs := []error{}
	for driverDirPathAbs, op := range prober.eventsMap {
		driverDirName := filepath.Base(driverDirPathAbs) // e.g. driverDirName = vendor~cifs
		probeEvent, pluginErr := prober.newProbeEvent(driverDirName, op)
		if pluginErr != nil {
			allErrs = append(allErrs, pluginErr)
			continue
		}
		probeEvents = append(probeEvents, probeEvent)

		delete(prober.eventsMap, driverDirPathAbs)
	}
	return probeEvents, errors.NewAggregate(allErrs)
}


func (prober *flexVolumeProber) probeAll() (events []volume.ProbeEvent, err error) {
	probeEvents := []volume.ProbeEvent{}
	allErrs := []error{}
	files, err := prober.fs.ReadDir(prober.pluginDir)
	if err != nil {
		return nil, fmt.Errorf("error reading the Flexvolume directory: %s", err)
	}
	for _, f := range files {
		// only directories with names that do not begin with '.' are counted as plugins
		// and pluginDir/dirname/dirname should be an executable
		// unless dirname contains '~' for escaping namespace
		// e.g. dirname = vendor~cifs
		// then, executable will be pluginDir/dirname/cifs
		if f.IsDir() && filepath.Base(f.Name())[0] != '.' {
			probeEvent, pluginErr := prober.newProbeEvent(f.Name(), volume.ProbeAddOrUpdate)
			if pluginErr != nil {
				allErrs = append(allErrs, pluginErr)
				continue
			}
			probeEvents = append(probeEvents, probeEvent)
		}
	}
	return probeEvents, errors.NewAggregate(allErrs)
}


func (prober *flexVolumeProber) newProbeEvent(driverDirName string, op volume.ProbeOperation) (volume.ProbeEvent, error) {
	probeEvent := volume.ProbeEvent{
		Op: op,
	}
	if op == volume.ProbeAddOrUpdate {
		plugin, pluginErr := prober.factory.NewFlexVolumePlugin(prober.pluginDir, driverDirName, prober.runner)
		if pluginErr != nil {
			pluginErr = fmt.Errorf(
				"error creating Flexvolume plugin from directory %s, skipping. Error: %s",
				driverDirName, pluginErr)
			return probeEvent, pluginErr
		}
		probeEvent.Plugin = plugin
		probeEvent.PluginName = plugin.GetPluginName()
	} else if op == volume.ProbeRemove {
		driverName := utilstrings.UnescapeQualifiedName(driverDirName)
		probeEvent.PluginName = driverName

	} else {
		return probeEvent, fmt.Errorf("Unknown Operation on directory: %s. ", driverDirName)
	}
	return probeEvent, nil
}


func (prober *flexVolumeProber) handleWatchEvent(event fsnotify.Event) error {
	// event.Name is the watched path.
	if filepath.Base(event.Name)[0] == '.' {
		// Ignore files beginning with '.'
		return nil
	}

	eventPathAbs, err := filepath.Abs(event.Name)
	if err != nil {
		return err
	}
	parentPathAbs := filepath.Dir(eventPathAbs)
	pluginDirAbs, err := filepath.Abs(prober.pluginDir)
	if err != nil {
		return err
	}

	// event of pluginDirAbs
	if eventPathAbs == pluginDirAbs {
		// If the Flexvolume plugin directory is removed, need to recreate it
		// in order to keep it under watch.
		if event.Has(fsnotify.Remove) {
			if err := prober.createPluginDir(); err != nil {
				return err
			}
			if err := prober.addWatchRecursive(pluginDirAbs); err != nil {
				return err
			}
		}
		return nil
	}

	// watch newly added subdirectories inside a driver directory
	if event.Has(fsnotify.Create) {
		if err := prober.addWatchRecursive(eventPathAbs); err != nil {
			return err
		}
	}

	eventRelPathToPluginDir, err := filepath.Rel(pluginDirAbs, eventPathAbs)
	if err != nil {
		return err
	}

	// event inside specific driver dir
	if len(eventRelPathToPluginDir) > 0 {
		driverDirName := strings.Split(eventRelPathToPluginDir, string(os.PathSeparator))[0]
		driverDirAbs := filepath.Join(pluginDirAbs, driverDirName)
		// executable is removed, will trigger ProbeRemove event
		if event.Has(fsnotify.Remove) && (eventRelPathToPluginDir == getExecutablePathRel(driverDirName) || parentPathAbs == pluginDirAbs) {
			prober.updateEventsMap(driverDirAbs, volume.ProbeRemove)
		} else {
			prober.updateEventsMap(driverDirAbs, volume.ProbeAddOrUpdate)
		}
	}

	return nil
}


// getExecutableName returns the executableName of a flex plugin
func getExecutablePathRel(driverDirName string) string {
	parts := strings.Split(driverDirName, "~")
	return filepath.Join(driverDirName, parts[len(parts)-1])
}


func (prober *flexVolumeProber) updateEventsMap(eventDirAbs string, op volume.ProbeOperation) {
	prober.mutex.Lock()
	defer prober.mutex.Unlock()
	if prober.probeAllNeeded {
		return
	}
	prober.eventsMap[eventDirAbs] = op
}


// Recursively adds to watch all directories inside and including the file specified by the given filename.
// If the file is a symlink to a directory, it will watch the symlink but not any of the subdirectories.
//
// Each file or directory change triggers two events: one from the watch on itself, another from the watch
// on its parent directory.
func (prober *flexVolumeProber) addWatchRecursive(filename string) error {
	addWatch := func(path string, info os.FileInfo, err error) error {
		if err == nil && info.IsDir() {
			if err := prober.watcher.AddWatch(path); err != nil {
				klog.Errorf("Error recursively adding watch: %v", err)
			}
		}
		return nil
	}
	return prober.fs.Walk(filename, addWatch)
}


// Creates a new filesystem watcher and adds watches for the plugin directory
// and all of its subdirectories.
func (prober *flexVolumeProber) initWatcher() error {
	err := prober.watcher.Init(func(event fsnotify.Event) {
		if err := prober.handleWatchEvent(event); err != nil {
			klog.Errorf("Flexvolume prober watch: %s", err)
		}
	}, func(err error) {
		klog.Errorf("Received an error from watcher: %s", err)
	})
	if err != nil {
		return fmt.Errorf("error initializing watcher: %s", err)
	}

	if err := prober.addWatchRecursive(prober.pluginDir); err != nil {
		return fmt.Errorf("error adding watch on Flexvolume directory: %s", err)
	}

	prober.watcher.Run()

	return nil
}


// Creates the plugin directory, if it doesn't already exist.
func (prober *flexVolumeProber) createPluginDir() error {
	if _, err := prober.fs.Stat(prober.pluginDir); os.IsNotExist(err) {
		klog.Warningf("Flexvolume plugin directory at %s does not exist. Recreating.", prober.pluginDir)
		err := prober.fs.MkdirAll(prober.pluginDir, 0755)
		if err != nil {
			return fmt.Errorf("error (re-)creating driver directory: %s", err)
		}
	}

	return nil
}


func (prober *flexVolumeProber) testAndSetProbeAllNeeded(newval bool) (oldval bool) {
	prober.mutex.Lock()
	defer prober.mutex.Unlock()
	oldval, prober.probeAllNeeded = prober.probeAllNeeded, newval
	return
}


func (f *unmounterDefaults) TearDownAt(dir string) error {
	klog.Warning(logPrefix(f.plugin), "using default TearDownAt for ", dir)
	return mount.CleanupMountPoint(dir, f.mounter, false)
}


// Unmounter interface
func (f *flexVolumeUnmounter) TearDown() error {
	path := f.GetPath()
	return f.TearDownAt(path)
}


func (f *flexVolumeUnmounter) TearDownAt(dir string) error {
	pathExists, pathErr := mount.PathExists(dir)
	if pathErr != nil {
		// only log warning here since plugins should anyways have to deal with errors
		klog.Warningf("Error checking path: %v", pathErr)
	} else {
		if !pathExists {
			klog.Warningf("Warning: Unmount skipped because path does not exist: %v", dir)
			return nil
		}
	}

	call := f.plugin.NewDriverCall(unmountCmd)
	call.Append(dir)
	_, err := call.Run()
	if isCmdNotSupportedErr(err) {
		err = (*unmounterDefaults)(f).TearDownAt(dir)
	}
	if err != nil {
		return err
	}

	// Flexvolume driver may remove the directory. Ignore if it does.
	if pathExists, pathErr := mount.PathExists(dir); pathErr != nil {
		return fmt.Errorf("error checking if path exists: %w", pathErr)
	} else if !pathExists {
		return nil
	}
	return os.Remove(dir)
}


func addSecretsToOptions(options map[string]string, spec *volume.Spec, namespace string, driverName string, host volume.VolumeHost) error {
	secretName, secretNamespace, err := getSecretNameAndNamespace(spec, namespace)
	if err != nil {
		return err
	}

	if len(secretName) == 0 || len(secretNamespace) == 0 {
		return nil
	}

	kubeClient := host.GetKubeClient()
	if kubeClient == nil {
		return fmt.Errorf("cannot get kube client")
	}

	secrets, err := util.GetSecretForPV(secretNamespace, secretName, driverName, host.GetKubeClient())
	if err != nil {
		err = fmt.Errorf("couldn't get secret %v/%v err: %w", secretNamespace, secretName, err)
		return err
	}
	for name, data := range secrets {
		options[optionKeySecret+"/"+name] = base64.StdEncoding.EncodeToString([]byte(data))
		klog.V(1).Infof("found flex volume secret info: %s", name)
	}

	return nil
}


func getDriver(spec *volume.Spec) (string, error) {
	if spec.Volume != nil && spec.Volume.FlexVolume != nil {
		return spec.Volume.FlexVolume.Driver, nil
	}
	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.FlexVolume != nil {
		return spec.PersistentVolume.Spec.FlexVolume.Driver, nil
	}
	return "", errNotFlexVolume
}


func getFSType(spec *volume.Spec) (string, error) {
	if spec.Volume != nil && spec.Volume.FlexVolume != nil {
		return spec.Volume.FlexVolume.FSType, nil
	}
	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.FlexVolume != nil {
		return spec.PersistentVolume.Spec.FlexVolume.FSType, nil
	}
	return "", errNotFlexVolume
}


func getSecretNameAndNamespace(spec *volume.Spec, podNamespace string) (string, string, error) {
	if spec.Volume != nil && spec.Volume.FlexVolume != nil {
		if spec.Volume.FlexVolume.SecretRef == nil {
			return "", "", nil
		}
		return spec.Volume.FlexVolume.SecretRef.Name, podNamespace, nil
	}
	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.FlexVolume != nil {
		if spec.PersistentVolume.Spec.FlexVolume.SecretRef == nil {
			return "", "", nil
		}
		secretName := spec.PersistentVolume.Spec.FlexVolume.SecretRef.Name
		secretNamespace := spec.PersistentVolume.Spec.FlexVolume.SecretRef.Namespace
		if len(secretNamespace) == 0 {
			secretNamespace = podNamespace
		}
		return secretName, secretNamespace, nil
	}
	return "", "", errNotFlexVolume
}


func getReadOnly(spec *volume.Spec) (bool, error) {
	if spec.Volume != nil && spec.Volume.FlexVolume != nil {
		return spec.Volume.FlexVolume.ReadOnly, nil
	}
	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.FlexVolume != nil {
		// ReadOnly is specified at the PV level
		return spec.ReadOnly, nil
	}
	return false, errNotFlexVolume
}


func getOptions(spec *volume.Spec) (map[string]string, error) {
	if spec.Volume != nil && spec.Volume.FlexVolume != nil {
		return spec.Volume.FlexVolume.Options, nil
	}
	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.FlexVolume != nil {
		return spec.PersistentVolume.Spec.FlexVolume.Options, nil
	}
	return nil, errNotFlexVolume
}


func prepareForMount(mounter mount.Interface, deviceMountPath string) (bool, error) {

	notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath)
	if err != nil {
		if os.IsNotExist(err) {
			if err := os.MkdirAll(deviceMountPath, 0750); err != nil {
				return false, err
			}
			notMnt = true
		} else {
			return false, err
		}
	}

	return !notMnt, nil
}


// Mounts the device at the given path.
// It is expected that prepareForMount has been called before.
func doMount(mounter mount.Interface, devicePath, deviceMountPath, fsType string, options []string) error {
	err := mounter.MountSensitiveWithoutSystemd(devicePath, deviceMountPath, fsType, options, nil)
	if err != nil {
		klog.Errorf("Failed to mount the volume at %s, device: %s, error: %s", deviceMountPath, devicePath, err.Error())
		return err
	}
	return nil
}


func isNotMounted(mounter mount.Interface, deviceMountPath string) (bool, error) {
	notmnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath)
	if err != nil {
		klog.Errorf("Error checking mount point %s, error: %v", deviceMountPath, err)
		return false, err
	}
	return notmnt, nil
}


func (f *flexVolume) GetPath() string

func (f *flexVolume) GetPath() string {
	name := f.driverName
	return f.plugin.host.GetPodVolumeDir(f.podUID, utilstrings.EscapeQualifiedName(name), f.volName)
}


func (plugin *gcePersistentDiskPlugin) NewAttacher() (volume.Attacher, error) {
	gceCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
	if err != nil {
		return nil, err
	}

	return &gcePersistentDiskAttacher{
		host:     plugin.host,
		gceDisks: gceCloud,
	}, nil
}


func (plugin *gcePersistentDiskPlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
	return plugin.NewAttacher()
}


func (plugin *gcePersistentDiskPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
	mounter := plugin.host.GetMounter(plugin.GetPluginName())
	return mounter.GetMountRefs(deviceMountPath)
}


// Attach checks with the GCE cloud provider if the specified volume is already
// attached to the node with the specified Name.
// If the volume is attached, it succeeds (returns nil).
// If it is not, Attach issues a call to the GCE cloud provider to attach it.
// Callers are responsible for retrying on failure.
// Callers are responsible for thread safety between concurrent attach and
// detach operations.
func (attacher *gcePersistentDiskAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) {
	volumeSource, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	pdName := volumeSource.PDName

	attached, err := attacher.gceDisks.DiskIsAttached(pdName, nodeName)
	if err != nil {
		// Log error and continue with attach
		klog.Errorf(
			"Error checking if PD (%q) is already attached to current node (%q). Will continue and try attach anyway. err=%v",
			pdName, nodeName, err)
	}

	if err == nil && attached {
		// Volume is already attached to node.
		klog.Infof("Attach operation is successful. PD %q is already attached to node %q.", pdName, nodeName)
	} else {
		if err := attacher.gceDisks.AttachDisk(pdName, nodeName, readOnly, isRegionalPD(spec)); err != nil {
			klog.Errorf("Error attaching PD %q to node %q: %+v", pdName, nodeName, err)
			return "", err
		}
	}

	return filepath.Join(diskByIDPath, diskGooglePrefix+pdName), nil
}


func (attacher *gcePersistentDiskAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.NodeName) (map[*volume.Spec]bool, error) {
	volumesAttachedCheck := make(map[*volume.Spec]bool)
	volumePdNameMap := make(map[string]*volume.Spec)
	pdNameList := []string{}
	for _, spec := range specs {
		volumeSource, _, err := getVolumeSource(spec)
		// If error is occurred, skip this volume and move to the next one
		if err != nil {
			klog.Errorf("Error getting volume (%q) source : %v", spec.Name(), err)
			continue
		}
		pdNameList = append(pdNameList, volumeSource.PDName)
		volumesAttachedCheck[spec] = true
		volumePdNameMap[volumeSource.PDName] = spec
	}
	attachedResult, err := attacher.gceDisks.DisksAreAttached(pdNameList, nodeName)
	if err != nil {
		// Log error and continue with attach
		klog.Errorf(
			"Error checking if PDs (%v) are already attached to current node (%q). err=%v",
			pdNameList, nodeName, err)
		return volumesAttachedCheck, err
	}

	for pdName, attached := range attachedResult {
		if !attached {
			spec := volumePdNameMap[pdName]
			volumesAttachedCheck[spec] = false
			klog.V(2).Infof("VolumesAreAttached: check volume %q (specName: %q) is no longer attached", pdName, spec.Name())
		}
	}
	return volumesAttachedCheck, nil
}


func (attacher *gcePersistentDiskAttacher) BulkVerifyVolumes(volumesByNode map[types.NodeName][]*volume.Spec) (map[types.NodeName]map[*volume.Spec]bool, error) {
	volumesAttachedCheck := make(map[types.NodeName]map[*volume.Spec]bool)
	diskNamesByNode := make(map[types.NodeName][]string)
	volumeSpecToDiskName := make(map[*volume.Spec]string)

	for nodeName, volumeSpecs := range volumesByNode {
		diskNames := []string{}
		for _, spec := range volumeSpecs {
			volumeSource, _, err := getVolumeSource(spec)
			if err != nil {
				klog.Errorf("Error getting volume (%q) source : %v", spec.Name(), err)
				continue
			}
			diskNames = append(diskNames, volumeSource.PDName)
			volumeSpecToDiskName[spec] = volumeSource.PDName
		}
		diskNamesByNode[nodeName] = diskNames
	}

	attachedDisksByNode, err := attacher.gceDisks.BulkDisksAreAttached(diskNamesByNode)
	if err != nil {
		return nil, err
	}

	for nodeName, volumeSpecs := range volumesByNode {
		volumesAreAttachedToNode := make(map[*volume.Spec]bool)
		for _, spec := range volumeSpecs {
			diskName := volumeSpecToDiskName[spec]
			volumesAreAttachedToNode[spec] = attachedDisksByNode[nodeName][diskName]
		}
		volumesAttachedCheck[nodeName] = volumesAreAttachedToNode
	}
	return volumesAttachedCheck, nil
}


// search Windows disk number by LUN
func getDiskID(pdName string, exec utilexec.Interface) (string, error) {
	// TODO: replace Get-GcePdName with native windows support of Get-Disk, see issue #74674
	cmd := `Get-GcePdName | select Name, DeviceId | ConvertTo-Json`
	output, err := exec.Command("powershell", "/c", cmd).CombinedOutput()
	if err != nil {
		klog.Errorf("Get-GcePdName failed, error: %v, output: %q", err, string(output))
		err = errors.New(err.Error() + " " + string(output))
		return "", err
	}

	var data []map[string]interface{}
	if err = json.Unmarshal(output, &data); err != nil {
		klog.Errorf("Get-Disk output is not a json array, output: %q", string(output))
		return "", err
	}

	for _, pd := range data {
		if jsonName, ok := pd["Name"]; ok {
			if name, ok := jsonName.(string); ok {
				if name == pdName {
					klog.Infof("found the disk %q", name)
					if diskNum, ok := pd["DeviceId"]; ok {
						switch v := diskNum.(type) {
						case int:
							return strconv.Itoa(v), nil
						case float64:
							return strconv.Itoa(int(v)), nil
						case string:
							return v, nil
						default:
							// diskNum isn't one of the types above
							klog.Warningf("Disk %q found, but disknumber (%q) is not in one of the recongnized type", name, diskNum)
						}
					}
				}
			}
		}
	}

	return "", fmt.Errorf("could not found disk number for disk %q", pdName)
}


func (attacher *gcePersistentDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) {
	ticker := time.NewTicker(checkSleepDuration)
	defer ticker.Stop()
	timer := time.NewTimer(timeout)
	defer timer.Stop()

	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	pdName := volumeSource.PDName

	if runtime.GOOS == "windows" {
		exec := attacher.host.GetExec(gcePersistentDiskPluginName)
		id, err := getDiskID(pdName, exec)
		if err != nil {
			klog.Errorf("WaitForAttach (windows) failed with error %s", err)
			return "", err
		}
		return id, nil
	}

	partition := ""
	if volumeSource.Partition != 0 {
		partition = strconv.Itoa(int(volumeSource.Partition))
	}

	sdBefore, err := filepath.Glob(diskSDPattern)
	if err != nil {
		klog.Errorf("Error filepath.Glob(\"%s\"): %v\r\n", diskSDPattern, err)
	}
	sdBeforeSet := sets.NewString(sdBefore...)

	devicePaths := getDiskByIDPaths(pdName, partition)
	for {
		select {
		case <-ticker.C:
			klog.V(5).Infof("Checking GCE PD %q is attached.", pdName)
			path, err := verifyDevicePath(devicePaths, sdBeforeSet, pdName)
			if err != nil {
				// Log error, if any, and continue checking periodically. See issue #11321
				klog.Errorf("Error verifying GCE PD (%q) is attached: %v", pdName, err)
			} else if path != "" {
				// A device path has successfully been created for the PD
				klog.Infof("Successfully found attached GCE PD %q.", pdName)
				return path, nil
			} else {
				klog.V(4).Infof("could not verify GCE PD (%q) is attached, device path does not exist", pdName)
			}
		case <-timer.C:
			return "", fmt.Errorf("could not find attached GCE PD %q. Timeout waiting for mount paths to be created", pdName)
		}
	}
}


func (attacher *gcePersistentDiskAttacher) GetDeviceMountPath(
	spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return makeGlobalPDName(attacher.host, volumeSource.PDName), nil
}


func (attacher *gcePersistentDiskAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string, _ volume.DeviceMounterArgs) error {
	// Only mount the PD globally once.
	mounter := attacher.host.GetMounter(gcePersistentDiskPluginName)
	notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath)
	if err != nil {
		if os.IsNotExist(err) {
			dir := deviceMountPath
			if runtime.GOOS == "windows" {
				// in windows, as we use mklink, only need to MkdirAll for parent directory
				dir = filepath.Dir(deviceMountPath)
			}
			if err := os.MkdirAll(dir, 0750); err != nil {
				return fmt.Errorf("MountDevice:CreateDirectory failed with %s", err)
			}
			notMnt = true
		} else {
			return err
		}
	}

	volumeSource, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return err
	}

	options := []string{}
	if readOnly {
		options = append(options, "ro")
	}
	if notMnt {
		diskMounter := volumeutil.NewSafeFormatAndMountFromHost(gcePersistentDiskPluginName, attacher.host)
		mountOptions := volumeutil.MountOptionFromSpec(spec, options...)
		err = diskMounter.FormatAndMount(devicePath, deviceMountPath, volumeSource.FSType, mountOptions)
		if err != nil {
			os.Remove(deviceMountPath)
			return err
		}
		klog.V(4).Infof("formatting spec %v devicePath %v deviceMountPath %v fs %v with options %+v", spec.Name(), devicePath, deviceMountPath, volumeSource.FSType, options)
	}
	return nil
}


func (plugin *gcePersistentDiskPlugin) NewDetacher() (volume.Detacher, error) {
	gceCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
	if err != nil {
		return nil, err
	}

	return &gcePersistentDiskDetacher{
		host:     plugin.host,
		gceDisks: gceCloud,
	}, nil
}


func (plugin *gcePersistentDiskPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
	return plugin.NewDetacher()
}


// Detach checks with the GCE cloud provider if the specified volume is already
// attached to the specified node. If the volume is not attached, it succeeds
// (returns nil). If it is attached, Detach issues a call to the GCE cloud
// provider to attach it.
// Callers are responsible for retrying on failure.
// Callers are responsible for thread safety between concurrent attach and detach
// operations.
func (detacher *gcePersistentDiskDetacher) Detach(volumeName string, nodeName types.NodeName) error {
	pdName := path.Base(volumeName)

	attached, err := detacher.gceDisks.DiskIsAttached(pdName, nodeName)
	if err != nil {
		// Log error and continue with detach
		klog.Errorf(
			"Error checking if PD (%q) is already attached to current node (%q). Will continue and try detach anyway. err=%v",
			pdName, nodeName, err)
	}

	if err == nil && !attached {
		// Volume is not attached to node. Success!
		klog.Infof("Detach operation is successful. PD %q was not attached to node %q.", pdName, nodeName)
		return nil
	}

	if err = detacher.gceDisks.DetachDisk(pdName, nodeName); err != nil {
		klog.Errorf("Error detaching PD %q from node %q: %v", pdName, nodeName, err)
		return err
	}

	return nil
}


func (detacher *gcePersistentDiskDetacher) UnmountDevice(deviceMountPath string) error {
	if runtime.GOOS == "windows" {
		// Flush data cache for windows because it does not do so automatically during unmount device
		exec := detacher.host.GetExec(gcePersistentDiskPluginName)
		err := volumeutil.WriteVolumeCache(deviceMountPath, exec)
		if err != nil {
			return err
		}
	}
	return mount.CleanupMountPoint(deviceMountPath, detacher.host.GetMounter(gcePersistentDiskPluginName), false)
}


func (plugin *gcePersistentDiskPlugin) CanAttach(spec *volume.Spec) (bool, error) {
	return true, nil
}


func (plugin *gcePersistentDiskPlugin) CanDeviceMount(spec *volume.Spec) (bool, error) {
	return true, nil
}


// ProbeVolumePlugins is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&gcePersistentDiskPlugin{nil}}
}


func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
	return host.GetPodVolumeDir(uid, utilstrings.EscapeQualifiedName(gcePersistentDiskPluginName), volName)
}


func (plugin *gcePersistentDiskPlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	return nil
}


func (plugin *gcePersistentDiskPlugin) GetPluginName() string {
	return gcePersistentDiskPluginName
}


func (plugin *gcePersistentDiskPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return volumeSource.PDName, nil
}


func (plugin *gcePersistentDiskPlugin) CanSupport(spec *volume.Spec) bool {
	return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.GCEPersistentDisk != nil) ||
		(spec.Volume != nil && spec.Volume.GCEPersistentDisk != nil)
}


func (plugin *gcePersistentDiskPlugin) RequiresRemount(spec *volume.Spec) bool {
	return false
}


func (plugin *gcePersistentDiskPlugin) SupportsMountOption() bool {
	return true
}


func (plugin *gcePersistentDiskPlugin) SupportsBulkVolumeVerification() bool {
	return true
}


func (plugin *gcePersistentDiskPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return false, nil
}


func (plugin *gcePersistentDiskPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
	return []v1.PersistentVolumeAccessMode{
		v1.ReadWriteOnce,
		v1.ReadOnlyMany,
	}
}


func (plugin *gcePersistentDiskPlugin) GetVolumeLimits() (map[string]int64, error) {
	volumeLimits := map[string]int64{
		util.GCEVolumeLimitKey: volumeLimitSmall,
	}
	cloud := plugin.host.GetCloudProvider()

	// if we can't fetch cloudprovider we return an error
	// hoping external CCM or admin can set it. Returning
	// default values from here will mean, no one can
	// override them.
	if cloud == nil {
		return nil, fmt.Errorf("no cloudprovider present")
	}

	if cloud.ProviderName() != gcecloud.ProviderName {
		return nil, fmt.Errorf("expected gce cloud got %s", cloud.ProviderName())
	}

	instances, ok := cloud.Instances()
	if !ok {
		klog.Warning("Failed to get instances from cloud provider")
		return volumeLimits, nil
	}

	instanceType, err := instances.InstanceType(context.TODO(), plugin.host.GetNodeName())
	if err != nil {
		klog.Errorf("Failed to get instance type from GCE cloud provider")
		return volumeLimits, nil
	}
	smallMachineTypes := []string{"f1-micro", "g1-small", "e2-micro", "e2-small", "e2-medium"}
	for _, small := range smallMachineTypes {
		if instanceType == small {
			volumeLimits[util.GCEVolumeLimitKey] = volumeLimitSmall
			return volumeLimits, nil
		}
	}
	volumeLimits[util.GCEVolumeLimitKey] = volumeLimitBig
	return volumeLimits, nil
}


func (plugin *gcePersistentDiskPlugin) VolumeLimitKey(spec *volume.Spec) string {
	return util.GCEVolumeLimitKey
}


func (plugin *gcePersistentDiskPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
	// Inject real implementations here, test through the internal function.
	return plugin.newMounterInternal(spec, pod.UID, &GCEDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName()))
}


func getVolumeSource(

func getVolumeSource(
	spec *volume.Spec) (*v1.GCEPersistentDiskVolumeSource, bool, error) {
	if spec.Volume != nil && spec.Volume.GCEPersistentDisk != nil {
		return spec.Volume.GCEPersistentDisk, spec.Volume.GCEPersistentDisk.ReadOnly, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.GCEPersistentDisk != nil {
		return spec.PersistentVolume.Spec.GCEPersistentDisk, spec.ReadOnly, nil
	}

	return nil, false, fmt.Errorf("spec does not reference a GCE volume type")
}


func (plugin *gcePersistentDiskPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager pdManager, mounter mount.Interface) (volume.Mounter, error) {
	// GCEPDs used directly in a pod have a ReadOnly flag set by the pod author.
	// GCEPDs used as a PersistentVolume gets the ReadOnly flag indirectly through the persistent-claim volume used to mount the PV
	volumeSource, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}

	pdName := volumeSource.PDName
	partition := ""
	if volumeSource.Partition != 0 {
		partition = strconv.Itoa(int(volumeSource.Partition))
	}

	return &gcePersistentDiskMounter{
		gcePersistentDisk: &gcePersistentDisk{
			podUID:          podUID,
			volName:         spec.Name(),
			pdName:          pdName,
			partition:       partition,
			mounter:         mounter,
			manager:         manager,
			plugin:          plugin,
			MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, spec.Name(), plugin.host)),
		},
		mountOptions: util.MountOptionFromSpec(spec),
		readOnly:     readOnly}, nil
}


func (plugin *gcePersistentDiskPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	// Inject real implementations here, test through the internal function.
	return plugin.newUnmounterInternal(volName, podUID, &GCEDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName()))
}


func (plugin *gcePersistentDiskPlugin) newUnmounterInternal(volName string, podUID types.UID, manager pdManager, mounter mount.Interface) (volume.Unmounter, error) {
	return &gcePersistentDiskUnmounter{&gcePersistentDisk{
		podUID:          podUID,
		volName:         volName,
		manager:         manager,
		mounter:         mounter,
		plugin:          plugin,
		MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)),
	}}, nil
}


func (plugin *gcePersistentDiskPlugin) NewDeleter(logger klog.Logger, spec *volume.Spec) (volume.Deleter, error) {
	return plugin.newDeleterInternal(spec, &GCEDiskUtil{})
}


func (plugin *gcePersistentDiskPlugin) newDeleterInternal(spec *volume.Spec, manager pdManager) (volume.Deleter, error) {
	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.GCEPersistentDisk == nil {
		return nil, fmt.Errorf("spec.PersistentVolumeSource.GCEPersistentDisk is nil")
	}
	return &gcePersistentDiskDeleter{
		gcePersistentDisk: &gcePersistentDisk{
			volName: spec.Name(),
			pdName:  spec.PersistentVolume.Spec.GCEPersistentDisk.PDName,
			manager: manager,
			plugin:  plugin,
		}}, nil
}


func (plugin *gcePersistentDiskPlugin) NewProvisioner(logger klog.Logger, options volume.VolumeOptions) (volume.Provisioner, error) {
	return plugin.newProvisionerInternal(options, &GCEDiskUtil{})
}


func (plugin *gcePersistentDiskPlugin) newProvisionerInternal(options volume.VolumeOptions, manager pdManager) (volume.Provisioner, error) {
	return &gcePersistentDiskProvisioner{
		gcePersistentDisk: &gcePersistentDisk{
			manager: manager,
			plugin:  plugin,
		},
		options: options,
	}, nil
}


func (plugin *gcePersistentDiskPlugin) RequiresFSResize() bool {
	return true
}


func (plugin *gcePersistentDiskPlugin) ExpandVolumeDevice(
	spec *volume.Spec,
	newSize resource.Quantity,
	oldSize resource.Quantity) (resource.Quantity, error) {
	cloud, err := getCloudProvider(plugin.host.GetCloudProvider())

	if err != nil {
		return oldSize, err
	}
	pdName := spec.PersistentVolume.Spec.GCEPersistentDisk.PDName
	updatedQuantity, err := cloud.ResizeDisk(pdName, oldSize, newSize)

	if err != nil {
		return oldSize, err
	}
	return updatedQuantity, nil
}


func (plugin *gcePersistentDiskPlugin) NodeExpand(resizeOptions volume.NodeResizeOptions) (bool, error) {
	fsVolume, err := util.CheckVolumeModeFilesystem(resizeOptions.VolumeSpec)
	if err != nil {
		return false, fmt.Errorf("error checking VolumeMode: %v", err)
	}
	// if volume is not a fs file system, there is nothing for us to do here.
	if !fsVolume {
		return true, nil
	}
	_, err = util.GenericResizeFS(plugin.host, plugin.GetPluginName(), resizeOptions.DevicePath, resizeOptions.DeviceMountPath)
	if err != nil {
		return false, err
	}
	return true, nil
}


func (plugin *gcePersistentDiskPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
	mounter := plugin.host.GetMounter(plugin.GetPluginName())
	kvh, ok := plugin.host.(volume.KubeletVolumeHost)
	if !ok {
		return volume.ReconstructedVolume{}, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
	}
	hu := kvh.GetHostUtil()
	pluginMntDir := util.GetPluginMountDir(plugin.host, plugin.GetPluginName())
	sourceName, err := hu.GetDeviceNameFromMount(mounter, mountPath, pluginMntDir)
	if err != nil {
		return volume.ReconstructedVolume{}, err
	}
	gceVolume := &v1.Volume{
		Name: volumeName,
		VolumeSource: v1.VolumeSource{
			GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
				PDName: sourceName,
			},
		},
	}
	return volume.ReconstructedVolume{
		Spec: volume.NewSpecFromVolume(gceVolume),
	}, nil
}


func (b *gcePersistentDiskMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:       b.readOnly,
		Managed:        !b.readOnly,
		SELinuxRelabel: true,
	}
}


// SetUp bind mounts the disk global mount to the volume path.
func (b *gcePersistentDiskMounter) SetUp(mounterArgs volume.MounterArgs) error {
	return b.SetUpAt(b.GetPath(), mounterArgs)
}


// SetUp bind mounts the disk global mount to the give volume path.
func (b *gcePersistentDiskMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	// TODO: handle failed mounts here.
	notMnt, err := b.mounter.IsLikelyNotMountPoint(dir)
	klog.V(4).Infof("GCE PersistentDisk set up: Dir (%s) PD name (%q) Mounted (%t) Error (%v), ReadOnly (%t)", dir, b.pdName, !notMnt, err, b.readOnly)
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("cannot validate mount point: %s %v", dir, err)
	}
	if !notMnt {
		return nil
	}

	if runtime.GOOS != "windows" {
		// in windows, we will use mklink to mount, will MkdirAll in Mount func
		if err := os.MkdirAll(dir, 0750); err != nil {
			return fmt.Errorf("mkdir failed on disk %s (%v)", dir, err)
		}
	}

	// Perform a bind mount to the full path to allow duplicate mounts of the same PD.
	options := []string{"bind"}
	if b.readOnly {
		options = append(options, "ro")
	}

	globalPDPath := makeGlobalPDName(b.plugin.host, b.pdName)
	klog.V(4).Infof("attempting to mount %s", dir)

	mountOptions := util.JoinMountOptions(b.mountOptions, options)

	err = b.mounter.MountSensitiveWithoutSystemd(globalPDPath, dir, "", mountOptions, nil)
	if err != nil {
		notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir)
		if mntErr != nil {
			return fmt.Errorf("failed to mount: %v. Cleanup IsLikelyNotMountPoint check failed: %v", err, mntErr)
		}
		if !notMnt {
			if mntErr = b.mounter.Unmount(dir); mntErr != nil {
				return fmt.Errorf("failed to mount: %v. Cleanup failed to unmount: %v", err, mntErr)
			}
			notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir)
			if mntErr != nil {
				return fmt.Errorf("failed to mount: %v. Cleanup IsLikelyNotMountPoint check failed: %v", err, mntErr)
			}
			if !notMnt {
				// This is very odd, we don't expect it.  We'll try again next sync loop.
				return fmt.Errorf("%s is still mounted, despite call to unmount().  Will try again next sync loop", dir)
			}
		}
		mntErr = os.Remove(dir)
		if mntErr != nil {
			return fmt.Errorf("failed to mount: %v. Cleanup os Remove(%s) failed: %v", err, dir, mntErr)
		}

		return fmt.Errorf("mount of disk %s failed: %v", dir, err)
	}

	klog.V(4).Infof("mount of disk %s succeeded", dir)
	if !b.readOnly {
		if err := volume.SetVolumeOwnership(b, dir, mounterArgs.FsGroup, mounterArgs.FSGroupChangePolicy, util.FSGroupCompleteHook(b.plugin, nil)); err != nil {
			klog.Errorf("SetVolumeOwnership returns error %v", err)
		}
	}
	return nil
}


func makeGlobalPDName(host volume.VolumeHost, devName string) string {
	return filepath.Join(host.GetPluginDir(gcePersistentDiskPluginName), util.MountsInGlobalPDPath, devName)
}


func (b *gcePersistentDiskMounter) GetPath() string {
	return getPath(b.podUID, b.volName, b.plugin.host)
}


func (c *gcePersistentDiskUnmounter) GetPath() string {
	return getPath(c.podUID, c.volName, c.plugin.host)
}


// Unmounts the bind mount, and detaches the disk only if the PD
// resource was the last reference to that disk on the kubelet.
func (c *gcePersistentDiskUnmounter) TearDown() error {
	return c.TearDownAt(c.GetPath())
}


// TearDownAt unmounts the bind mount
func (c *gcePersistentDiskUnmounter) TearDownAt(dir string) error {
	return mount.CleanupMountPoint(dir, c.mounter, false)
}


func (d *gcePersistentDiskDeleter) GetPath() string {
	return getPath(d.podUID, d.volName, d.plugin.host)
}


func (d *gcePersistentDiskDeleter) Delete() error {
	return d.manager.DeleteVolume(d)
}


func (c *gcePersistentDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologies []v1.TopologySelectorTerm) (*v1.PersistentVolume, error) {
	if !util.ContainsAllAccessModes(c.plugin.GetAccessModes(), c.options.PVC.Spec.AccessModes) {
		return nil, fmt.Errorf("invalid AccessModes %v: only AccessModes %v are supported", c.options.PVC.Spec.AccessModes, c.plugin.GetAccessModes())
	}

	volumeID, sizeGB, labels, fstype, err := c.manager.CreateVolume(c, selectedNode, allowedTopologies)
	if err != nil {
		return nil, err
	}

	if fstype == "" {
		fstype = "ext4"
	}

	volumeMode := c.options.PVC.Spec.VolumeMode
	if volumeMode != nil && *volumeMode == v1.PersistentVolumeBlock {
		// Block volumes should not have any FSType
		fstype = ""
	}

	pv := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name:   c.options.PVName,
			Labels: map[string]string{},
			Annotations: map[string]string{
				util.VolumeDynamicallyCreatedByKey: "gce-pd-dynamic-provisioner",
			},
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeReclaimPolicy: c.options.PersistentVolumeReclaimPolicy,
			AccessModes:                   c.options.PVC.Spec.AccessModes,
			Capacity: v1.ResourceList{
				v1.ResourceName(v1.ResourceStorage): resource.MustParse(fmt.Sprintf("%dGi", sizeGB)),
			},
			VolumeMode: volumeMode,
			PersistentVolumeSource: v1.PersistentVolumeSource{
				GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
					PDName:    volumeID,
					Partition: 0,
					ReadOnly:  false,
					FSType:    fstype,
				},
			},
			MountOptions: c.options.MountOptions,
		},
	}
	if len(c.options.PVC.Spec.AccessModes) == 0 {
		pv.Spec.AccessModes = c.plugin.GetAccessModes()
	}

	requirements := make([]v1.NodeSelectorRequirement, 0)
	if len(labels) != 0 {
		if pv.Labels == nil {
			pv.Labels = make(map[string]string)
		}
		for k, v := range labels {
			pv.Labels[k] = v
			var values []string
			if k == v1.LabelTopologyZone {
				values, err = volumehelpers.LabelZonesToList(v)
				if err != nil {
					return nil, fmt.Errorf("failed to convert label string for Zone: %s to a List: %v", v, err)
				}
			} else {
				values = []string{v}
			}
			requirements = append(requirements, v1.NodeSelectorRequirement{Key: k, Operator: v1.NodeSelectorOpIn, Values: values})
		}
	}

	if len(requirements) > 0 {
		pv.Spec.NodeAffinity = new(v1.VolumeNodeAffinity)
		pv.Spec.NodeAffinity.Required = new(v1.NodeSelector)
		pv.Spec.NodeAffinity.Required.NodeSelectorTerms = make([]v1.NodeSelectorTerm, 1)
		pv.Spec.NodeAffinity.Required.NodeSelectorTerms[0].MatchExpressions = requirements
	}

	return pv, nil
}


func (plugin *gcePersistentDiskPlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mapPath string) (*volume.Spec, error) {
	pluginDir := plugin.host.GetVolumeDevicePluginDir(gcePersistentDiskPluginName)
	blkutil := volumepathhandler.NewBlockVolumePathHandler()
	globalMapPathUUID, err := blkutil.FindGlobalMapPathUUIDFromPod(pluginDir, mapPath, podUID)
	if err != nil {
		return nil, err
	}
	klog.V(5).Infof("globalMapPathUUID: %v, err: %v", globalMapPathUUID, err)

	globalMapPath := filepath.Dir(globalMapPathUUID)
	if len(globalMapPath) <= 1 {
		return nil, fmt.Errorf("failed to get volume plugin information from globalMapPathUUID: %v", globalMapPathUUID)
	}

	return getVolumeSpecFromGlobalMapPath(volumeName, globalMapPath)
}


func getVolumeSpecFromGlobalMapPath(volumeName, globalMapPath string) (*volume.Spec, error) {
	// Get volume spec information from globalMapPath
	// globalMapPath example:
	//   plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumeID}
	//   plugins/kubernetes.io/gce-pd/volumeDevices/vol-XXXXXX
	pdName := filepath.Base(globalMapPath)
	if len(pdName) <= 1 {
		return nil, fmt.Errorf("failed to get pd name from global path=%s", globalMapPath)
	}
	block := v1.PersistentVolumeBlock
	gceVolume := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: volumeName,
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeSource: v1.PersistentVolumeSource{
				GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
					PDName: pdName,
				},
			},
			VolumeMode: &block,
		},
	}

	return volume.NewSpecFromPersistentVolume(gceVolume, true), nil
}


// NewBlockVolumeMapper creates a new volume.BlockVolumeMapper from an API specification.
func (plugin *gcePersistentDiskPlugin) NewBlockVolumeMapper(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.BlockVolumeMapper, error) {
	// If this is called via GenerateUnmapDeviceFunc(), pod is nil.
	// Pass empty string as dummy uid since uid isn't used in the case.
	var uid types.UID
	if pod != nil {
		uid = pod.UID
	}

	return plugin.newBlockVolumeMapperInternal(spec, uid, &GCEDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName()))
}


func (plugin *gcePersistentDiskPlugin) newBlockVolumeMapperInternal(spec *volume.Spec, podUID types.UID, manager pdManager, mounter mount.Interface) (volume.BlockVolumeMapper, error) {
	volumeSource, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}
	pdName := volumeSource.PDName
	partition := ""
	if volumeSource.Partition != 0 {
		partition = strconv.Itoa(int(volumeSource.Partition))
	}

	mapper := &gcePersistentDiskMapper{
		gcePersistentDisk: &gcePersistentDisk{
			volName:   spec.Name(),
			podUID:    podUID,
			pdName:    pdName,
			partition: partition,
			manager:   manager,
			mounter:   mounter,
			plugin:    plugin,
		},
		readOnly: readOnly,
	}

	blockPath, err := mapper.GetGlobalMapPath(spec)
	if err != nil {
		return nil, fmt.Errorf("failed to get device path: %v", err)
	}
	mapper.MetricsProvider = volume.NewMetricsBlock(filepath.Join(blockPath, string(podUID)))

	return mapper, nil
}


func (plugin *gcePersistentDiskPlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (volume.BlockVolumeUnmapper, error) {
	return plugin.newUnmapperInternal(volName, podUID, &GCEDiskUtil{})
}


func (plugin *gcePersistentDiskPlugin) newUnmapperInternal(volName string, podUID types.UID, manager pdManager) (volume.BlockVolumeUnmapper, error) {
	return &gcePersistentDiskUnmapper{
		gcePersistentDisk: &gcePersistentDisk{
			volName: volName,
			podUID:  podUID,
			pdName:  volName,
			manager: manager,
			plugin:  plugin,
		}}, nil
}


// GetGlobalMapPath returns global map path and error
// path: plugins/kubernetes.io/{PluginName}/volumeDevices/pdName
func (pd *gcePersistentDisk) GetGlobalMapPath(spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}
	return filepath.Join(pd.plugin.host.GetVolumeDevicePluginDir(gcePersistentDiskPluginName), string(volumeSource.PDName)), nil
}


// GetPodDeviceMapPath returns pod device map path and volume name
// path: pods/{podUid}/volumeDevices/kubernetes.io~aws
func (pd *gcePersistentDisk) GetPodDeviceMapPath() (string, string) {
	name := gcePersistentDiskPluginName
	return pd.plugin.host.GetPodVolumeDeviceDir(pd.podUID, utilstrings.EscapeQualifiedName(name)), pd.volName
}


// SupportsMetrics returns true for gcePersistentDisk as it initializes the
// MetricsProvider.
func (pd *gcePersistentDisk) SupportsMetrics() bool {
	return true
}


// DeleteVolume deletes a GCE PD
// Returns: error
func (util *GCEDiskUtil) DeleteVolume(d *gcePersistentDiskDeleter) error {
	cloud, err := getCloudProvider(d.gcePersistentDisk.plugin.host.GetCloudProvider())
	if err != nil {
		return err
	}

	if err = cloud.DeleteDisk(d.pdName); err != nil {
		klog.V(2).Infof("Error deleting GCE PD volume %s: %v", d.pdName, err)
		// GCE cloud provider returns volume.deletedVolumeInUseError when
		// necessary, no handling needed here.
		return err
	}
	klog.V(2).Infof("Successfully deleted GCE PD volume %s", d.pdName)
	return nil
}


// CreateVolume creates a GCE PD.
// Returns: gcePDName, volumeSizeGB, labels, fsType, error
func (util *GCEDiskUtil) CreateVolume(c *gcePersistentDiskProvisioner, node *v1.Node, allowedTopologies []v1.TopologySelectorTerm) (string, int, map[string]string, string, error) {
	cloud, err := getCloudProvider(c.gcePersistentDisk.plugin.host.GetCloudProvider())
	if err != nil {
		return "", 0, nil, "", err
	}

	name := volumeutil.GenerateVolumeName(c.options.ClusterName, c.options.PVName, 63) // GCE PD name can have up to 63 characters
	capacity := c.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	// GCE PDs are allocated in chunks of GiBs
	requestGB, err := volumehelpers.RoundUpToGiB(capacity)
	if err != nil {
		return "", 0, nil, "", err
	}

	// Apply Parameters.
	// Values for parameter "replication-type" are canonicalized to lower case.
	// Values for other parameters are case-insensitive, and we leave validation of these values
	// to the cloud provider.
	diskType := ""
	configuredZone := ""
	var configuredZones sets.String
	zonePresent := false
	zonesPresent := false
	replicationType := replicationTypeNone
	fstype := ""
	for k, v := range c.options.Parameters {
		switch strings.ToLower(k) {
		case "type":
			diskType = v
		case "zone":
			zonePresent = true
			configuredZone = v
		case "zones":
			zonesPresent = true
			configuredZones, err = volumehelpers.ZonesToSet(v)
			if err != nil {
				return "", 0, nil, "", err
			}
		case "replication-type":
			replicationType = strings.ToLower(v)
		case volume.VolumeParameterFSType:
			fstype = v
		default:
			return "", 0, nil, "", fmt.Errorf("invalid option %q for volume plugin %s", k, c.plugin.GetPluginName())
		}
	}

	// TODO: implement PVC.Selector parsing
	if c.options.PVC.Spec.Selector != nil {
		return "", 0, nil, "", fmt.Errorf("claim.Spec.Selector is not supported for dynamic provisioning on GCE")
	}

	var activezones sets.String
	activezones, err = cloud.GetAllCurrentZones()
	if err != nil {
		return "", 0, nil, "", err
	}

	var disk *gcecloud.Disk
	switch replicationType {
	case replicationTypeRegionalPD:
		selectedZones, err := volumehelpers.SelectZonesForVolume(zonePresent, zonesPresent, configuredZone, configuredZones, activezones, node, allowedTopologies, c.options.PVC.Name, maxRegionalPDZones)
		if err != nil {
			klog.V(2).Infof("Error selecting zones for regional GCE PD volume: %v", err)
			return "", 0, nil, "", err
		}
		disk, err = cloud.CreateRegionalDisk(
			name,
			diskType,
			selectedZones,
			requestGB,
			*c.options.CloudTags)
		if err != nil {
			klog.V(2).Infof("Error creating regional GCE PD volume: %v", err)
			return "", 0, nil, "", err
		}
		klog.V(2).Infof("Successfully created Regional GCE PD volume %s", name)

	case replicationTypeNone:
		selectedZone, err := volumehelpers.SelectZoneForVolume(zonePresent, zonesPresent, configuredZone, configuredZones, activezones, node, allowedTopologies, c.options.PVC.Name)
		if err != nil {
			return "", 0, nil, "", err
		}
		disk, err = cloud.CreateDisk(
			name,
			diskType,
			selectedZone,
			requestGB,
			*c.options.CloudTags)
		if err != nil {
			klog.V(2).Infof("Error creating single-zone GCE PD volume: %v", err)
			return "", 0, nil, "", err
		}
		klog.V(2).Infof("Successfully created single-zone GCE PD volume %s", name)

	default:
		return "", 0, nil, "", fmt.Errorf("replication-type of '%s' is not supported", replicationType)
	}

	labels, err := cloud.GetAutoLabelsForPD(disk)
	if err != nil {
		// We don't really want to leak the volume here...
		klog.Errorf("error getting labels for volume %q: %v", name, err)
	}

	return name, int(requestGB), labels, fstype, nil
}


// Returns the first path that exists, or empty string if none exist.
func verifyDevicePath(devicePaths []string, sdBeforeSet sets.String, diskName string) (string, error) {
	if err := udevadmChangeToNewDrives(sdBeforeSet); err != nil {
		// It's possible udevadm was called on other disks so it should not block this
		// call. If it did fail on this disk, then the devicePath will either
		// not exist or be wrong. If it's wrong, then the scsi_id check below will fail.
		klog.Errorf("udevadmChangeToNewDrives failed with: %v", err)
	}

	for _, path := range devicePaths {
		if pathExists, err := mount.PathExists(path); err != nil {
			return "", fmt.Errorf("error checking if path exists: %v", err)
		} else if pathExists {
			// validate that the path actually resolves to the correct disk
			serial, err := getScsiSerial(path, diskName)
			if err != nil {
				return "", fmt.Errorf("failed to get scsi serial %v", err)
			}
			if serial != diskName {
				// The device link is not pointing to the correct device
				// Trigger udev on this device to try to fix the link
				if udevErr := udevadmChangeToDrive(path); udevErr != nil {
					klog.Errorf("udevadmChangeToDrive %q failed with: %v", path, err)
				}

				// Return error to retry WaitForAttach and verifyDevicePath
				return "", fmt.Errorf("scsi_id serial %q for device %q doesn't match disk %q", serial, path, diskName)
			}
			// The device link is correct
			return path, nil
		}
	}

	return "", nil
}


// Calls scsi_id on the given devicePath to get the serial number reported by that device.
func getScsiSerial(devicePath, diskName string) (string, error) {
	exists, err := utilpath.Exists(utilpath.CheckFollowSymlink, "/lib/udev/scsi_id")
	if err != nil {
		return "", fmt.Errorf("failed to check scsi_id existence: %v", err)
	}

	if !exists {
		klog.V(6).Infof("scsi_id doesn't exist; skipping check for %v", devicePath)
		return diskName, nil
	}

	out, err := exec.New().Command(
		"/lib/udev/scsi_id",
		"--page=0x83",
		"--whitelisted",
		fmt.Sprintf("--device=%v", devicePath)).CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("scsi_id failed for device %q with %v", devicePath, err)
	}

	return parseScsiSerial(string(out))
}


// Parse the output returned by scsi_id and extract the serial number
func parseScsiSerial(output string) (string, error) {
	substrings := scsiRegex.FindStringSubmatch(output)
	if substrings == nil {
		return "", fmt.Errorf("scsi_id output cannot be parsed: %q", output)
	}

	return substrings[1], nil
}


// Returns list of all /dev/disk/by-id/* paths for given PD.
func getDiskByIDPaths(pdName string, partition string) []string {
	devicePaths := []string{
		filepath.Join(diskByIDPath, diskGooglePrefix+pdName),
		filepath.Join(diskByIDPath, diskScsiGooglePrefix+pdName),
	}

	if partition != "" {
		for i, path := range devicePaths {
			devicePaths[i] = path + diskPartitionSuffix + partition
		}
	}

	return devicePaths
}


// Return cloud provider
func getCloudProvider(cloudProvider cloudprovider.Interface) (*gcecloud.Cloud, error) {
	var err error
	for numRetries := 0; numRetries < maxRetries; numRetries++ {
		gceCloudProvider, ok := cloudProvider.(*gcecloud.Cloud)
		if !ok || gceCloudProvider == nil {
			// Retry on error. See issue #11321
			klog.Errorf("Failed to get GCE Cloud Provider. plugin.host.GetCloudProvider returned %v instead", cloudProvider)
			time.Sleep(errorSleepDuration)
			continue
		}

		return gceCloudProvider, nil
	}

	return nil, fmt.Errorf("failed to get GCE GCECloudProvider with error %v", err)
}


// Triggers the application of udev rules by calling "udevadm trigger
// --action=change" for newly created "/dev/sd*" drives (exist only in
// after set). This is workaround for Issue #7972. Once the underlying
// issue has been resolved, this may be removed.
func udevadmChangeToNewDrives(sdBeforeSet sets.String) error {
	sdAfter, err := filepath.Glob(diskSDPattern)
	if err != nil {
		return fmt.Errorf("error filepath.Glob(\"%s\"): %v\r", diskSDPattern, err)
	}

	for _, sd := range sdAfter {
		if !sdBeforeSet.Has(sd) {
			return udevadmChangeToDrive(sd)
		}
	}

	return nil
}


// Calls "udevadm trigger --action=change" on the specified drive.
// drivePath must be the block device path to trigger on, in the format "/dev/sd*", or a symlink to it.
// This is workaround for Issue #7972. Once the underlying issue has been resolved, this may be removed.
func udevadmChangeToDrive(drivePath string) error {
	klog.V(5).Infof("udevadmChangeToDrive: drive=%q", drivePath)

	// Evaluate symlink, if any
	drive, err := filepath.EvalSymlinks(drivePath)
	if err != nil {
		return fmt.Errorf("udevadmChangeToDrive: filepath.EvalSymlinks(%q) failed with %v", drivePath, err)
	}
	klog.V(5).Infof("udevadmChangeToDrive: symlink path is %q", drive)

	// Check to make sure input is "/dev/sd*"
	if !strings.Contains(drive, diskSDPath) {
		return fmt.Errorf("udevadmChangeToDrive: expected input in the form \"%s\" but drive is %q", diskSDPattern, drive)
	}

	// Call "udevadm trigger --action=change --property-match=DEVNAME=/dev/sd..."
	_, err = exec.New().Command(
		"udevadm",
		"trigger",
		"--action=change",
		fmt.Sprintf("--property-match=DEVNAME=%s", drive)).CombinedOutput()
	if err != nil {
		return fmt.Errorf("udevadmChangeToDrive: udevadm trigger failed for drive %q with %v", drive, err)
	}
	return nil
}


// Checks whether the given GCE PD volume spec is associated with a regional PD.
func isRegionalPD(spec *volume.Spec) bool {
	if spec.PersistentVolume != nil {
		zonesLabel := spec.PersistentVolume.Labels[v1.LabelTopologyZone]
		if zonesLabel == "" {
			zonesLabel = spec.PersistentVolume.Labels[v1.LabelFailureDomainBetaZone]
		}
		zones := strings.Split(zonesLabel, cloudvolume.LabelMultiZoneDelimiter)
		return len(zones) > 1
	}
	return false
}


// This is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&gitRepoPlugin{nil}}
}


func wrappedVolumeSpec() volume.Spec

func wrappedVolumeSpec() volume.Spec {
	return volume.Spec{
		Volume: &v1.Volume{VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}},
	}
}


func (plugin *gitRepoPlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	return nil
}


func (plugin *gitRepoPlugin) GetPluginName() string {
	return gitRepoPluginName
}


func (plugin *gitRepoPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	volumeSource, _ := getVolumeSource(spec)
	if volumeSource == nil {
		return "", fmt.Errorf("Spec does not reference a Git repo volume type")
	}

	return fmt.Sprintf(
		"%v:%v:%v",
		volumeSource.Repository,
		volumeSource.Revision,
		volumeSource.Directory), nil
}


func (plugin *gitRepoPlugin) CanSupport(spec *volume.Spec) bool {
	return spec.Volume != nil && spec.Volume.GitRepo != nil
}


func (plugin *gitRepoPlugin) RequiresRemount(spec *volume.Spec) bool {
	return false
}


func (plugin *gitRepoPlugin) SupportsMountOption() bool {
	return false
}


func (plugin *gitRepoPlugin) SupportsBulkVolumeVerification() bool {
	return false
}


func (plugin *gitRepoPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return false, nil
}


func (plugin *gitRepoPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) {
	if err := validateVolume(spec.Volume.GitRepo); err != nil {
		return nil, err
	}

	return &gitRepoVolumeMounter{
		gitRepoVolume: &gitRepoVolume{
			volName: spec.Name(),
			podUID:  pod.UID,
			plugin:  plugin,
		},
		pod:      *pod,
		source:   spec.Volume.GitRepo.Repository,
		revision: spec.Volume.GitRepo.Revision,
		target:   spec.Volume.GitRepo.Directory,
		exec:     exec.New(),
		opts:     opts,
	}, nil
}


func (plugin *gitRepoPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	return &gitRepoVolumeUnmounter{
		&gitRepoVolume{
			volName: volName,
			podUID:  podUID,
			plugin:  plugin,
		},
	}, nil
}


func (plugin *gitRepoPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
	gitVolume := &v1.Volume{
		Name: volumeName,
		VolumeSource: v1.VolumeSource{
			GitRepo: &v1.GitRepoVolumeSource{},
		},
	}
	return volume.ReconstructedVolume{
		Spec: volume.NewSpecFromVolume(gitVolume),
	}, nil
}


func (gr *gitRepoVolume) GetPath() string {
	name := gitRepoPluginName
	return gr.plugin.host.GetPodVolumeDir(gr.podUID, utilstrings.EscapeQualifiedName(name), gr.volName)
}


func (b *gitRepoVolumeMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:       false,
		Managed:        true,
		SELinuxRelabel: true, // xattr change should be okay, TODO: double check
	}
}


// SetUp creates new directory and clones a git repo.
func (b *gitRepoVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error {
	return b.SetUpAt(b.GetPath(), mounterArgs)
}


// SetUpAt creates new directory and clones a git repo.
func (b *gitRepoVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	if volumeutil.IsReady(b.getMetaDir()) {
		return nil
	}

	// Wrap EmptyDir, let it do the setup.
	wrapped, err := b.plugin.host.NewWrapperMounter(b.volName, wrappedVolumeSpec(), &b.pod, b.opts)
	if err != nil {
		return err
	}
	if err := wrapped.SetUpAt(dir, mounterArgs); err != nil {
		return err
	}

	args := []string{"clone", "--", b.source}

	if len(b.target) != 0 {
		args = append(args, b.target)
	}
	if output, err := b.execCommand("git", args, dir); err != nil {
		return fmt.Errorf("failed to exec 'git %s': %s: %v",
			strings.Join(args, " "), output, err)
	}

	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return err
	}

	if len(b.revision) == 0 {
		// Done!
		volumeutil.SetReady(b.getMetaDir())
		return nil
	}

	var subdir string

	switch {
	case len(b.target) != 0 && filepath.Clean(b.target) == ".":
		// if target dir is '.', use the current dir
		subdir = filepath.Join(dir)
	case len(files) == 1:
		// if target is not '.', use the generated folder
		subdir = filepath.Join(dir, files[0].Name())
	default:
		// if target is not '.', but generated many files, it's wrong
		return fmt.Errorf("unexpected directory contents: %v", files)
	}

	if output, err := b.execCommand("git", []string{"checkout", b.revision}, subdir); err != nil {
		return fmt.Errorf("failed to exec 'git checkout %s': %s: %v", b.revision, output, err)
	}
	if output, err := b.execCommand("git", []string{"reset", "--hard"}, subdir); err != nil {
		return fmt.Errorf("failed to exec 'git reset --hard': %s: %v", output, err)
	}

	volume.SetVolumeOwnership(b, dir, mounterArgs.FsGroup, nil /*fsGroupChangePolicy*/, volumeutil.FSGroupCompleteHook(b.plugin, nil))

	volumeutil.SetReady(b.getMetaDir())
	return nil
}


func (b *gitRepoVolumeMounter) getMetaDir() string {
	return filepath.Join(b.plugin.host.GetPodPluginDir(b.podUID, utilstrings.EscapeQualifiedName(gitRepoPluginName)), b.volName)
}


func (b *gitRepoVolumeMounter) execCommand(command string, args []string, dir string) ([]byte, error) {
	cmd := b.exec.Command(command, args...)
	cmd.SetDir(dir)
	return cmd.CombinedOutput()
}


func validateVolume(src *v1.GitRepoVolumeSource) error {
	if err := validateNonFlagArgument(src.Repository, "repository"); err != nil {
		return err
	}
	if err := validateNonFlagArgument(src.Revision, "revision"); err != nil {
		return err
	}
	if err := validateNonFlagArgument(src.Directory, "directory"); err != nil {
		return err
	}
	return nil
}


// TearDown simply deletes everything in the directory.
func (c *gitRepoVolumeUnmounter) TearDown() error {
	return c.TearDownAt(c.GetPath())
}


// TearDownAt simply deletes everything in the directory.
func (c *gitRepoVolumeUnmounter) TearDownAt(dir string) error {
	return volumeutil.UnmountViaEmptyDir(dir, c.plugin.host, c.volName, wrappedVolumeSpec(), c.podUID)
}


func getVolumeSource(spec *volume.Spec) (*v1.GitRepoVolumeSource, bool) {
	var readOnly bool
	var volumeSource *v1.GitRepoVolumeSource

	if spec.Volume != nil && spec.Volume.GitRepo != nil {
		volumeSource = spec.Volume.GitRepo
		readOnly = spec.ReadOnly
	}

	return volumeSource, readOnly
}


func validateNonFlagArgument(arg, argName string) error {
	if len(arg) > 0 && arg[0] == '-' {
		return fmt.Errorf("%q is an invalid value for %s", arg, argName)
	}
	return nil
}


// ProbeVolumePlugins is the primary entrypoint for volume plugins.
// The volumeConfig arg provides the ability to configure volume behavior.  It is implemented as a pointer to allow nils.
// The hostPathPlugin is used to store the volumeConfig and give it, when needed, to the func that Recycles.
// Tests that exercise recycling should not use this func but instead use ProbeRecyclablePlugins() to override default behavior.
func ProbeVolumePlugins(volumeConfig volume.VolumeConfig) []volume.VolumePlugin {
	return []volume.VolumePlugin{
		&hostPathPlugin{
			host:   nil,
			config: volumeConfig,
		},
	}
}


func FakeProbeVolumePlugins(volumeConfig volume.VolumeConfig) []volume.VolumePlugin {
	return []volume.VolumePlugin{
		&hostPathPlugin{
			host:          nil,
			config:        volumeConfig,
			noTypeChecker: true,
		},
	}
}


func (plugin *hostPathPlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	return nil
}


func (plugin *hostPathPlugin) GetPluginName() string {
	return hostPathPluginName
}


func (plugin *hostPathPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return volumeSource.Path, nil
}


func (plugin *hostPathPlugin) CanSupport(spec *volume.Spec) bool {
	return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.HostPath != nil) ||
		(spec.Volume != nil && spec.Volume.HostPath != nil)
}


func (plugin *hostPathPlugin) RequiresRemount(spec *volume.Spec) bool {
	return false
}


func (plugin *hostPathPlugin) SupportsMountOption() bool {
	return false
}


func (plugin *hostPathPlugin) SupportsBulkVolumeVerification() bool {
	return false
}


func (plugin *hostPathPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return false, nil
}


func (plugin *hostPathPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
	return []v1.PersistentVolumeAccessMode{
		v1.ReadWriteOnce,
	}
}


func (plugin *hostPathPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) {
	hostPathVolumeSource, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}

	path := hostPathVolumeSource.Path
	pathType := new(v1.HostPathType)
	if hostPathVolumeSource.Type == nil {
		*pathType = v1.HostPathUnset
	} else {
		pathType = hostPathVolumeSource.Type
	}
	kvh, ok := plugin.host.(volume.KubeletVolumeHost)
	if !ok {
		return nil, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
	}
	return &hostPathMounter{
		hostPath:      &hostPath{path: path, pathType: pathType},
		readOnly:      readOnly,
		mounter:       plugin.host.GetMounter(plugin.GetPluginName()),
		hu:            kvh.GetHostUtil(),
		noTypeChecker: plugin.noTypeChecker,
	}, nil
}


func (plugin *hostPathPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	return &hostPathUnmounter{&hostPath{
		path: "",
	}}, nil
}


// Recycle recycles/scrubs clean a HostPath volume.
// Recycle blocks until the pod has completed or any error occurs.
// HostPath recycling only works in single node clusters and is meant for testing purposes only.
func (plugin *hostPathPlugin) Recycle(pvName string, spec *volume.Spec, eventRecorder recyclerclient.RecycleEventRecorder) error {
	if spec.PersistentVolume == nil || spec.PersistentVolume.Spec.HostPath == nil {
		return fmt.Errorf("spec.PersistentVolume.Spec.HostPath is nil")
	}

	pod := plugin.config.RecyclerPodTemplate
	timeout := util.CalculateTimeoutForVolume(plugin.config.RecyclerMinimumTimeout, plugin.config.RecyclerTimeoutIncrement, spec.PersistentVolume)
	// overrides
	pod.Spec.ActiveDeadlineSeconds = &timeout
	pod.Spec.Volumes[0].VolumeSource = v1.VolumeSource{
		HostPath: &v1.HostPathVolumeSource{
			Path: spec.PersistentVolume.Spec.HostPath.Path,
		},
	}
	return recyclerclient.RecycleVolumeByWatchingPodUntilCompletion(pvName, pod, plugin.host.GetKubeClient(), eventRecorder)
}


func (plugin *hostPathPlugin) NewDeleter(logger klog.Logger, spec *volume.Spec) (volume.Deleter, error) {
	return newDeleter(spec, plugin.host)
}


func (plugin *hostPathPlugin) NewProvisioner(logger klog.Logger, options volume.VolumeOptions) (volume.Provisioner, error) {
	if !plugin.config.ProvisioningEnabled {
		return nil, fmt.Errorf("provisioning in volume plugin %q is disabled", plugin.GetPluginName())
	}
	return newProvisioner(options, plugin.host, plugin)
}


func (plugin *hostPathPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
	hostPathVolume := &v1.Volume{
		Name: volumeName,
		VolumeSource: v1.VolumeSource{
			HostPath: &v1.HostPathVolumeSource{
				Path: volumeName,
			},
		},
	}
	return volume.ReconstructedVolume{
		Spec: volume.NewSpecFromVolume(hostPathVolume),
	}, nil
}


func newDeleter(spec *volume.Spec, host volume.VolumeHost) (volume.Deleter, error) {
	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.HostPath == nil {
		return nil, fmt.Errorf("spec.PersistentVolumeSource.HostPath is nil")
	}
	path := spec.PersistentVolume.Spec.HostPath.Path
	return &hostPathDeleter{name: spec.Name(), path: path, host: host}, nil
}


func newProvisioner(options volume.VolumeOptions, host volume.VolumeHost, plugin *hostPathPlugin) (volume.Provisioner, error) {
	return &hostPathProvisioner{options: options, host: host, plugin: plugin, basePath: "hostpath_pv"}, nil
}


func (hp *hostPath) GetPath() string

func (hp *hostPath) GetPath() string {
	return hp.path
}


func (b *hostPathMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:       b.readOnly,
		Managed:        false,
		SELinuxRelabel: false,
	}
}


// SetUp does nothing.
func (b *hostPathMounter) SetUp(mounterArgs volume.MounterArgs) error {
	err := validation.ValidatePathNoBacksteps(b.GetPath())
	if err != nil {
		return fmt.Errorf("invalid HostPath `%s`: %v", b.GetPath(), err)
	}

	if *b.pathType == v1.HostPathUnset {
		return nil
	}
	if b.noTypeChecker {
		return nil
	} else {
		return checkType(b.GetPath(), b.pathType, b.hu)
	}
}


// SetUpAt does not make sense for host paths - probably programmer error.
func (b *hostPathMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	return fmt.Errorf("SetUpAt() does not make sense for host paths")
}


func (b *hostPathMounter) GetPath() string {
	return b.path
}


// TearDown does nothing.
func (c *hostPathUnmounter) TearDown() error {
	return nil
}


// TearDownAt does not make sense for host paths - probably programmer error.
func (c *hostPathUnmounter) TearDownAt(dir string) error {
	return fmt.Errorf("TearDownAt() does not make sense for host paths")
}


// Create for hostPath simply creates a local /tmp/%/%s directory as a new PersistentVolume, default /tmp/hostpath_pv/%s.
// This Provisioner is meant for development and testing only and WILL NOT WORK in a multi-node cluster.
func (r *hostPathProvisioner) Provision(selectedNode *v1.Node, allowedTopologies []v1.TopologySelectorTerm) (*v1.PersistentVolume, error) {
	if util.CheckPersistentVolumeClaimModeBlock(r.options.PVC) {
		return nil, fmt.Errorf("%s does not support block volume provisioning", r.plugin.GetPluginName())
	}

	fullpath := fmt.Sprintf("/tmp/%s/%s", r.basePath, uuid.NewUUID())

	capacity := r.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	pv := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: r.options.PVName,
			Annotations: map[string]string{
				util.VolumeDynamicallyCreatedByKey: "hostpath-dynamic-provisioner",
			},
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeReclaimPolicy: r.options.PersistentVolumeReclaimPolicy,
			AccessModes:                   r.options.PVC.Spec.AccessModes,
			Capacity: v1.ResourceList{
				v1.ResourceName(v1.ResourceStorage): capacity,
			},
			PersistentVolumeSource: v1.PersistentVolumeSource{
				HostPath: &v1.HostPathVolumeSource{
					Path: fullpath,
				},
			},
		},
	}
	if len(r.options.PVC.Spec.AccessModes) == 0 {
		pv.Spec.AccessModes = r.plugin.GetAccessModes()
	}

	if err := os.MkdirAll(pv.Spec.HostPath.Path, 0750); err != nil {
		return nil, err
	}
	if selinux.GetEnabled() {
		err := selinux.SetFileLabel(pv.Spec.HostPath.Path, config.KubeletContainersSharedSELinuxLabel)
		if err != nil {
			return nil, fmt.Errorf("failed to set selinux label for %q: %v", pv.Spec.HostPath.Path, err)
		}
	}

	return pv, nil
}


func (r *hostPathDeleter) GetPath() string {
	return r.path
}


// Delete for hostPath removes the local directory so long as it is beneath /tmp/*.
// THIS IS FOR TESTING AND LOCAL DEVELOPMENT ONLY!  This message should scare you away from using
// this deleter for anything other than development and testing.
func (r *hostPathDeleter) Delete() error {
	regexp := regexp.MustCompile("/tmp/.+")
	if !regexp.MatchString(r.GetPath()) {
		return fmt.Errorf("host_path deleter only supports /tmp/.+ but received provided %s", r.GetPath())
	}
	return os.RemoveAll(r.GetPath())
}


func getVolumeSource(spec *volume.Spec) (*v1.HostPathVolumeSource, bool, error) {
	if spec.Volume != nil && spec.Volume.HostPath != nil {
		return spec.Volume.HostPath, spec.ReadOnly, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.HostPath != nil {
		return spec.PersistentVolume.Spec.HostPath, spec.ReadOnly, nil
	}

	return nil, false, fmt.Errorf("spec does not reference an HostPath volume type")
}


func (ftc *fileTypeChecker) Exists() bool {
	exists, err := ftc.hu.PathExists(ftc.path)
	return exists && err == nil
}


func (ftc *fileTypeChecker) IsFile() bool {
	if !ftc.Exists() {
		return false
	}
	pathType, err := ftc.hu.GetFileType(ftc.path)
	if err != nil {
		return false
	}
	return string(pathType) == string(v1.HostPathFile)
}


func (ftc *fileTypeChecker) MakeFile() error {
	return makeFile(ftc.path)
}


func (ftc *fileTypeChecker) IsDir() bool {
	if !ftc.Exists() {
		return false
	}
	pathType, err := ftc.hu.GetFileType(ftc.path)
	if err != nil {
		return false
	}
	return string(pathType) == string(v1.HostPathDirectory)
}


func (ftc *fileTypeChecker) MakeDir() error {
	return makeDir(ftc.path)
}


func (ftc *fileTypeChecker) IsBlock() bool {
	blkDevType, err := ftc.hu.GetFileType(ftc.path)
	if err != nil {
		return false
	}
	return string(blkDevType) == string(v1.HostPathBlockDev)
}


func (ftc *fileTypeChecker) IsChar() bool {
	charDevType, err := ftc.hu.GetFileType(ftc.path)
	if err != nil {
		return false
	}
	return string(charDevType) == string(v1.HostPathCharDev)
}


func (ftc *fileTypeChecker) IsSocket() bool {
	socketType, err := ftc.hu.GetFileType(ftc.path)
	if err != nil {
		return false
	}
	return string(socketType) == string(v1.HostPathSocket)
}


func (ftc *fileTypeChecker) GetPath() string {
	return ftc.path
}


func newFileTypeChecker(path string, hu hostutil.HostUtils) hostPathTypeChecker {
	return &fileTypeChecker{path: path, hu: hu}
}


// checkType checks whether the given path is the exact pathType
func checkType(path string, pathType *v1.HostPathType, hu hostutil.HostUtils) error {
	return checkTypeInternal(newFileTypeChecker(path, hu), pathType)
}


func checkTypeInternal(ftc hostPathTypeChecker, pathType *v1.HostPathType) error {
	switch *pathType {
	case v1.HostPathDirectoryOrCreate:
		if !ftc.Exists() {
			return ftc.MakeDir()
		}
		fallthrough
	case v1.HostPathDirectory:
		if !ftc.IsDir() {
			return fmt.Errorf("hostPath type check failed: %s is not a directory", ftc.GetPath())
		}
	case v1.HostPathFileOrCreate:
		if !ftc.Exists() {
			return ftc.MakeFile()
		}
		fallthrough
	case v1.HostPathFile:
		if !ftc.IsFile() {
			return fmt.Errorf("hostPath type check failed: %s is not a file", ftc.GetPath())
		}
	case v1.HostPathSocket:
		if !ftc.IsSocket() {
			return fmt.Errorf("hostPath type check failed: %s is not a socket file", ftc.GetPath())
		}
	case v1.HostPathCharDev:
		if !ftc.IsChar() {
			return fmt.Errorf("hostPath type check failed: %s is not a character device", ftc.GetPath())
		}
	case v1.HostPathBlockDev:
		if !ftc.IsBlock() {
			return fmt.Errorf("hostPath type check failed: %s is not a block device", ftc.GetPath())
		}
	default:
		return fmt.Errorf("%s is an invalid volume type", *pathType)
	}

	return nil
}


func makeDir(pathname string) error

// makeDir creates a new directory.
// If pathname already exists as a directory, no error is returned.
// If pathname already exists as a file, an error is returned.
func makeDir(pathname string) error {
	err := os.MkdirAll(pathname, os.FileMode(0755))
	if err != nil {
		if !os.IsExist(err) {
			return err
		}
	}
	return nil
}


func makeFile(pathname string) error

// makeFile creates an empty file.
// If pathname already exists, whether a file or directory, no error is returned.
func makeFile(pathname string) error {
	f, err := os.OpenFile(pathname, os.O_CREATE, os.FileMode(0644))
	if f != nil {
		f.Close()
	}
	if err != nil {
		if !os.IsExist(err) {
			return err
		}
	}
	return nil
}


func (plugin *iscsiPlugin) NewAttacher() (volume.Attacher, error) {
	return &iscsiAttacher{
		host:        plugin.host,
		targetLocks: plugin.targetLocks,
		manager:     &ISCSIUtil{},
	}, nil
}


func (plugin *iscsiPlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
	return plugin.NewAttacher()
}


func (plugin *iscsiPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
	mounter := plugin.host.GetMounter(iscsiPluginName)
	return mounter.GetMountRefs(deviceMountPath)
}


func (attacher *iscsiAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) {
	return "", nil
}


func (attacher *iscsiAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.NodeName) (map[*volume.Spec]bool, error) {
	volumesAttachedCheck := make(map[*volume.Spec]bool)
	for _, spec := range specs {
		volumesAttachedCheck[spec] = true
	}

	return volumesAttachedCheck, nil
}


func (attacher *iscsiAttacher) WaitForAttach(spec *volume.Spec, devicePath string, pod *v1.Pod, timeout time.Duration) (string, error) {
	mounter, err := volumeSpecToMounter(spec, attacher.host, attacher.targetLocks, pod)
	if err != nil {
		klog.Warningf("failed to get iscsi mounter: %v", err)
		return "", err
	}
	return attacher.manager.AttachDisk(*mounter)
}


func (attacher *iscsiAttacher) GetDeviceMountPath(
	spec *volume.Spec) (string, error) {
	mounter, err := volumeSpecToMounter(spec, attacher.host, attacher.targetLocks, nil)
	if err != nil {
		klog.Warningf("failed to get iscsi mounter: %v", err)
		return "", err
	}
	if mounter.InitiatorName != "" {
		// new iface name is <target portal>:<volume name>
		mounter.Iface = mounter.Portals[0] + ":" + mounter.VolName
	}
	return attacher.manager.MakeGlobalPDName(*mounter.iscsiDisk), nil
}


func (attacher *iscsiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string, mountArgs volume.DeviceMounterArgs) error {
	mounter := attacher.host.GetMounter(iscsiPluginName)
	notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath)
	if err != nil {
		if os.IsNotExist(err) {
			if err := os.MkdirAll(deviceMountPath, 0750); err != nil {
				return err
			}
			notMnt = true
		} else {
			return err
		}
	}
	readOnly, fsType, err := getISCSIVolumeInfo(spec)
	if err != nil {
		return err
	}

	options := []string{}
	if readOnly {
		options = append(options, "ro")
	}
	if mountArgs.SELinuxLabel != "" {
		options = volumeutil.AddSELinuxMountOption(options, mountArgs.SELinuxLabel)
	}
	if notMnt {
		diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Exec: attacher.host.GetExec(iscsiPluginName)}
		mountOptions := volumeutil.MountOptionFromSpec(spec, options...)
		err = diskMounter.FormatAndMount(devicePath, deviceMountPath, fsType, mountOptions)
		if err != nil {
			os.Remove(deviceMountPath)
			return err
		}
	}
	return nil
}


func (plugin *iscsiPlugin) NewDetacher() (volume.Detacher, error) {
	return &iscsiDetacher{
		host:    plugin.host,
		mounter: plugin.host.GetMounter(iscsiPluginName),
		manager: &ISCSIUtil{},
		plugin:  plugin,
	}, nil
}


func (plugin *iscsiPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
	return plugin.NewDetacher()
}


func (detacher *iscsiDetacher) Detach(volumeName string, nodeName types.NodeName) error {
	return nil
}


func (detacher *iscsiDetacher) UnmountDevice(deviceMountPath string) error {
	unMounter := volumeSpecToUnmounter(detacher.mounter, detacher.host, detacher.plugin)
	err := detacher.manager.DetachDisk(*unMounter, deviceMountPath)
	if err != nil {
		return fmt.Errorf("iscsi: failed to detach disk: %s\nError: %v", deviceMountPath, err)
	}
	klog.V(4).Infof("iscsi: %q is unmounted, deleting the directory", deviceMountPath)
	err = os.RemoveAll(deviceMountPath)
	if err != nil {
		return fmt.Errorf("iscsi: failed to delete the directory: %s\nError: %v", deviceMountPath, err)
	}
	klog.V(4).Infof("iscsi: successfully detached disk: %s", deviceMountPath)
	return nil
}


func (plugin *iscsiPlugin) CanAttach(spec *volume.Spec) (bool, error) {
	return true, nil
}


func (plugin *iscsiPlugin) CanDeviceMount(spec *volume.Spec) (bool, error) {
	return true, nil
}


func volumeSpecToMounter(spec *volume.Spec, host volume.VolumeHost, targetLocks keymutex.KeyMutex, pod *v1.Pod) (*iscsiDiskMounter, error) {
	var secret map[string]string
	readOnly, fsType, err := getISCSIVolumeInfo(spec)
	if err != nil {
		return nil, err
	}
	var podUID types.UID
	if pod != nil {
		secret, err = createSecretMap(spec, &iscsiPlugin{host: host, targetLocks: targetLocks}, pod.Namespace)
		if err != nil {
			return nil, err
		}
		podUID = pod.UID
	}
	iscsiDisk, err := createISCSIDisk(spec,
		podUID,
		&iscsiPlugin{host: host, targetLocks: targetLocks},
		&ISCSIUtil{},
		secret,
	)
	if err != nil {
		return nil, err
	}
	exec := host.GetExec(iscsiPluginName)

	volumeMode, err := volumeutil.GetVolumeMode(spec)
	if err != nil {
		return nil, err
	}

	klog.V(5).Infof("iscsi: VolumeSpecToMounter volumeMode %s", volumeMode)
	return &iscsiDiskMounter{
		iscsiDisk:  iscsiDisk,
		fsType:     fsType,
		volumeMode: volumeMode,
		readOnly:   readOnly,
		mounter:    &mount.SafeFormatAndMount{Interface: host.GetMounter(iscsiPluginName), Exec: exec},
		exec:       exec,
		deviceUtil: volumeutil.NewDeviceHandler(volumeutil.NewIOHandler()),
	}, nil
}


func volumeSpecToUnmounter(mounter mount.Interface, host volume.VolumeHost, plugin *iscsiPlugin) *iscsiDiskUnmounter {
	exec := host.GetExec(iscsiPluginName)
	return &iscsiDiskUnmounter{
		iscsiDisk: &iscsiDisk{
			plugin: plugin,
		},
		mounter:    mounter,
		exec:       exec,
		deviceUtil: volumeutil.NewDeviceHandler(volumeutil.NewIOHandler()),
	}
}


// utility to mount a disk based filesystem
// globalPDPath: global mount path like, /var/lib/kubelet/plugins/kubernetes.io/iscsi/{ifaceName}/{portal-some_iqn-lun-lun_id}
// volPath: pod volume dir path like, /var/lib/kubelet/pods/{podUID}/volumes/kubernetes.io~iscsi/{volumeName}
func diskSetUp(manager diskManager, b iscsiDiskMounter, volPath string, mounter mount.Interface, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy) error {
	notMnt, err := mounter.IsLikelyNotMountPoint(volPath)
	if err != nil && !os.IsNotExist(err) {
		klog.Errorf("cannot validate mountpoint: %s", volPath)
		return err
	}
	if !notMnt {
		return nil
	}

	if err := os.MkdirAll(volPath, 0750); err != nil {
		klog.Errorf("failed to mkdir:%s", volPath)
		return err
	}
	// Perform a bind mount to the full path to allow duplicate mounts of the same disk.
	options := []string{"bind"}
	if b.readOnly {
		options = append(options, "ro")
	}
	if b.iscsiDisk.InitiatorName != "" {
		// new iface name is <target portal>:<volume name>
		b.iscsiDisk.Iface = b.iscsiDisk.Portals[0] + ":" + b.iscsiDisk.VolName
	}
	globalPDPath := manager.MakeGlobalPDName(*b.iscsiDisk)
	mountOptions := util.JoinMountOptions(b.mountOptions, options)
	err = mounter.MountSensitiveWithoutSystemd(globalPDPath, volPath, "", mountOptions, nil)
	if err != nil {
		klog.Errorf("Failed to bind mount: source:%s, target:%s, err:%v", globalPDPath, volPath, err)
		noMnt, mntErr := b.mounter.IsLikelyNotMountPoint(volPath)
		if mntErr != nil {
			klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
			return err
		}
		if !noMnt {
			if mntErr = b.mounter.Unmount(volPath); mntErr != nil {
				klog.Errorf("Failed to unmount: %v", mntErr)
				return err
			}
			noMnt, mntErr = b.mounter.IsLikelyNotMountPoint(volPath)
			if mntErr != nil {
				klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
				return err
			}
			if !noMnt {
				//  will most likely retry on next sync loop.
				klog.Errorf("%s is still mounted, despite call to unmount().  Will try again next sync loop.", volPath)
				return err
			}
		}
		os.Remove(volPath)
		return err
	}

	if !b.readOnly {
		volume.SetVolumeOwnership(&b, volPath, fsGroup, fsGroupChangePolicy, util.FSGroupCompleteHook(b.plugin, nil))
	}

	return nil
}


// ProbeVolumePlugins is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&iscsiPlugin{}}
}


func (plugin *iscsiPlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	plugin.targetLocks = keymutex.NewHashed(0)
	return nil
}


func (plugin *iscsiPlugin) GetPluginName() string {
	return iscsiPluginName
}


func (plugin *iscsiPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	tp, _, iqn, lun, err := getISCSITargetInfo(spec)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%v:%v:%v", tp, iqn, lun), nil
}


func (plugin *iscsiPlugin) CanSupport(spec *volume.Spec) bool {
	return (spec.Volume != nil && spec.Volume.ISCSI != nil) || (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.ISCSI != nil)
}


func (plugin *iscsiPlugin) RequiresRemount(spec *volume.Spec) bool {
	return false
}


func (plugin *iscsiPlugin) SupportsMountOption() bool {
	return true
}


func (plugin *iscsiPlugin) SupportsBulkVolumeVerification() bool {
	return false
}


func (plugin *iscsiPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return true, nil
}


func (plugin *iscsiPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
	return []v1.PersistentVolumeAccessMode{
		v1.ReadWriteOnce,
		v1.ReadOnlyMany,
	}
}


func (plugin *iscsiPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
	if pod == nil {
		return nil, fmt.Errorf("nil pod")
	}
	secret, err := createSecretMap(spec, plugin, pod.Namespace)
	if err != nil {
		return nil, err
	}
	return plugin.newMounterInternal(spec, pod.UID, &ISCSIUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()), secret)
}


func (plugin *iscsiPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface, exec utilexec.Interface, secret map[string]string) (volume.Mounter, error) {
	readOnly, fsType, err := getISCSIVolumeInfo(spec)
	if err != nil {
		return nil, err
	}
	iscsiDisk, err := createISCSIDisk(spec, podUID, plugin, manager, secret)
	if err != nil {
		return nil, err
	}

	if iscsiDisk != nil {

		//Add volume metrics
		iscsiDisk.MetricsProvider = volume.NewMetricsStatFS(iscsiDisk.GetPath())
	}
	return &iscsiDiskMounter{
		iscsiDisk:    iscsiDisk,
		fsType:       fsType,
		readOnly:     readOnly,
		mounter:      &mount.SafeFormatAndMount{Interface: mounter, Exec: exec},
		exec:         exec,
		deviceUtil:   ioutil.NewDeviceHandler(ioutil.NewIOHandler()),
		mountOptions: ioutil.MountOptionFromSpec(spec),
	}, nil
}


// NewBlockVolumeMapper creates a new volume.BlockVolumeMapper from an API specification.
func (plugin *iscsiPlugin) NewBlockVolumeMapper(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.BlockVolumeMapper, error) {
	// If this is called via GenerateUnmapDeviceFunc(), pod is nil.
	// Pass empty string as dummy uid since uid isn't used in the case.
	var uid types.UID
	var secret map[string]string
	var err error
	if pod != nil {
		uid = pod.UID
		secret, err = createSecretMap(spec, plugin, pod.Namespace)
		if err != nil {
			return nil, err
		}
	}
	return plugin.newBlockVolumeMapperInternal(spec, uid, &ISCSIUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()), secret)
}


func (plugin *iscsiPlugin) newBlockVolumeMapperInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface, exec utilexec.Interface, secret map[string]string) (volume.BlockVolumeMapper, error) {
	readOnly, _, err := getISCSIVolumeInfo(spec)
	if err != nil {
		return nil, err
	}
	iscsiDisk, err := createISCSIDisk(spec, podUID, plugin, manager, secret)
	if err != nil {
		return nil, err
	}
	mapper := &iscsiDiskMapper{
		iscsiDisk:  iscsiDisk,
		readOnly:   readOnly,
		exec:       exec,
		deviceUtil: ioutil.NewDeviceHandler(ioutil.NewIOHandler()),
	}

	blockPath, err := mapper.GetGlobalMapPath(spec)
	if err != nil {
		return nil, fmt.Errorf("failed to get device path: %v", err)
	}
	mapper.MetricsProvider = volume.NewMetricsBlock(filepath.Join(blockPath, string(podUID)))

	return mapper, nil
}


func (plugin *iscsiPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	return plugin.newUnmounterInternal(volName, podUID, &ISCSIUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()))
}


func (plugin *iscsiPlugin) newUnmounterInternal(volName string, podUID types.UID, manager diskManager, mounter mount.Interface, exec utilexec.Interface) (volume.Unmounter, error) {
	return &iscsiDiskUnmounter{
		iscsiDisk: &iscsiDisk{
			podUID:          podUID,
			VolName:         volName,
			manager:         manager,
			plugin:          plugin,
			MetricsProvider: volume.NewMetricsStatFS(plugin.host.GetPodVolumeDir(podUID, utilstrings.EscapeQualifiedName(iscsiPluginName), volName)),
		},
		mounter:    mounter,
		exec:       exec,
		deviceUtil: ioutil.NewDeviceHandler(ioutil.NewIOHandler()),
	}, nil
}


// NewBlockVolumeUnmapper creates a new volume.BlockVolumeUnmapper from recoverable state.
func (plugin *iscsiPlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (volume.BlockVolumeUnmapper, error) {
	return plugin.newUnmapperInternal(volName, podUID, &ISCSIUtil{}, plugin.host.GetExec(plugin.GetPluginName()))
}


func (plugin *iscsiPlugin) newUnmapperInternal(volName string, podUID types.UID, manager diskManager, exec utilexec.Interface) (volume.BlockVolumeUnmapper, error) {
	return &iscsiDiskUnmapper{
		iscsiDisk: &iscsiDisk{
			podUID:  podUID,
			VolName: volName,
			manager: manager,
			plugin:  plugin,
		},
		exec:       exec,
		deviceUtil: ioutil.NewDeviceHandler(ioutil.NewIOHandler()),
	}, nil
}


func (plugin *iscsiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
	// Find globalPDPath from pod volume directory(mountPath)
	var globalPDPath string
	mounter := plugin.host.GetMounter(plugin.GetPluginName())
	// Try really hard to get the global mount of the volume, an error returned from here would
	// leave the global mount still mounted, while marking the volume as unused.
	// The volume can then be mounted on several nodes, resulting in volume
	// corruption.
	paths, err := ioutil.GetReliableMountRefs(mounter, mountPath)
	if io.IsInconsistentReadError(err) {
		klog.Errorf("Failed to read mount refs from /proc/mounts for %s: %s", mountPath, err)
		klog.Errorf("Kubelet cannot unmount volume at %s, please unmount it and all mounts of the same device manually.", mountPath)
		return volume.ReconstructedVolume{}, err
	}
	if err != nil {
		return volume.ReconstructedVolume{}, err
	}

	for _, path := range paths {
		if strings.Contains(path, plugin.host.GetPluginDir(iscsiPluginName)) {
			globalPDPath = path
			break
		}
	}
	// Couldn't fetch globalPDPath
	if len(globalPDPath) == 0 {
		return volume.ReconstructedVolume{}, fmt.Errorf("couldn't fetch globalPDPath. failed to obtain volume spec")
	}

	// Obtain iscsi disk configurations from globalPDPath
	device, _, err := extractDeviceAndPrefix(globalPDPath)
	if err != nil {
		return volume.ReconstructedVolume{}, err
	}
	bkpPortal, iqn, err := extractPortalAndIqn(device)
	if err != nil {
		return volume.ReconstructedVolume{}, err
	}
	arr := strings.Split(device, "-lun-")
	if len(arr) < 2 {
		return volume.ReconstructedVolume{}, fmt.Errorf("failed to retrieve lun from globalPDPath: %v", globalPDPath)
	}
	lun, err := strconv.Atoi(arr[1])
	if err != nil {
		return volume.ReconstructedVolume{}, err
	}
	iface, _ := extractIface(globalPDPath)
	iscsiVolume := &v1.Volume{
		Name: volumeName,
		VolumeSource: v1.VolumeSource{
			ISCSI: &v1.ISCSIVolumeSource{
				TargetPortal:   bkpPortal,
				IQN:            iqn,
				Lun:            int32(lun),
				ISCSIInterface: iface,
			},
		},
	}

	var mountContext string
	if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) {
		kvh, ok := plugin.host.(volume.KubeletVolumeHost)
		if !ok {
			return volume.ReconstructedVolume{}, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
		}
		hu := kvh.GetHostUtil()
		mountContext, err = hu.GetSELinuxMountContext(mountPath)
		if err != nil {
			return volume.ReconstructedVolume{}, err
		}
	}

	return volume.ReconstructedVolume{
		Spec:                volume.NewSpecFromVolume(iscsiVolume),
		SELinuxMountContext: mountContext,
	}, nil
}


func (plugin *iscsiPlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mapPath string) (*volume.Spec, error) {
	pluginDir := plugin.host.GetVolumeDevicePluginDir(iscsiPluginName)
	blkutil := volumepathhandler.NewBlockVolumePathHandler()
	globalMapPathUUID, err := blkutil.FindGlobalMapPathUUIDFromPod(pluginDir, mapPath, podUID)
	if err != nil {
		return nil, err
	}
	klog.V(5).Infof("globalMapPathUUID: %v, err: %v", globalMapPathUUID, err)
	// Retrieve volume information from globalMapPathUUID
	// globalMapPathUUID example:
	// plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{pod uuid}
	// plugins/kubernetes.io/iscsi/volumeDevices/iface-default/192.168.0.10:3260-iqn.2017-05.com.example:test-lun-0/{pod uuid}
	globalMapPath := filepath.Dir(globalMapPathUUID)
	return getVolumeSpecFromGlobalMapPath(volumeName, globalMapPath)
}


func (iscsi *iscsiDisk) GetPath() string {
	name := iscsiPluginName
	// safe to use PodVolumeDir now: volume teardown occurs before pod is cleaned up
	return iscsi.plugin.host.GetPodVolumeDir(iscsi.podUID, utilstrings.EscapeQualifiedName(name), iscsi.VolName)
}


func (iscsi *iscsiDisk) iscsiGlobalMapPath(spec *volume.Spec) (string, error) {
	mounter, err := volumeSpecToMounter(spec, iscsi.plugin.host, iscsi.plugin.targetLocks, nil /* pod */)
	if err != nil {
		klog.Warningf("failed to get iscsi mounter: %v", err)
		return "", err
	}
	return iscsi.manager.MakeGlobalVDPDName(*mounter.iscsiDisk), nil
}


func (iscsi *iscsiDisk) iscsiPodDeviceMapPath() (string, string) {
	name := iscsiPluginName
	return iscsi.plugin.host.GetPodVolumeDeviceDir(iscsi.podUID, utilstrings.EscapeQualifiedName(name)), iscsi.VolName
}


func (b *iscsiDiskMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:       b.readOnly,
		Managed:        !b.readOnly,
		SELinuxRelabel: !b.mountedWithSELinuxContext,
	}
}


func (b *iscsiDiskMounter) SetUp(mounterArgs volume.MounterArgs) error {
	return b.SetUpAt(b.GetPath(), mounterArgs)
}


func (b *iscsiDiskMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	// diskSetUp checks mountpoints and prevent repeated calls
	err := diskSetUp(b.manager, *b, dir, b.mounter, mounterArgs.FsGroup, mounterArgs.FSGroupChangePolicy)
	if err != nil {
		klog.Errorf("iscsi: failed to setup")
	}

	if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) {
		// The volume must have been mounted in MountDevice with -o context.
		// TODO: extract from mount table in GetAttributes() to be sure?
		b.mountedWithSELinuxContext = mounterArgs.SELinuxLabel != ""
	}
	return err
}


// Unmounts the bind mount, and detaches the disk only if the disk
// resource was the last reference to that disk on the kubelet.
func (c *iscsiDiskUnmounter) TearDown() error {
	return c.TearDownAt(c.GetPath())
}


func (c *iscsiDiskUnmounter) TearDownAt(dir string) error {
	return mount.CleanupMountPoint(dir, c.mounter, false)
}


// SupportsMetrics returns true for SupportsMetrics as it initializes the
// MetricsProvider.
func (idm *iscsiDiskMapper) SupportsMetrics() bool {
	return true
}


// Even though iSCSI plugin has attacher/detacher implementation, iSCSI plugin
// needs volume detach operation during TearDownDevice(). This method is only
// chance that operations are done on kubelet node during volume teardown sequences.
func (c *iscsiDiskUnmapper) TearDownDevice(mapPath, _ string) error {
	err := c.manager.DetachBlockISCSIDisk(*c, mapPath)
	if err != nil {
		return fmt.Errorf("iscsi: failed to detach disk: %s\nError: %v", mapPath, err)
	}
	klog.V(4).Infof("iscsi: %q is unmounted, deleting the directory", mapPath)
	err = os.RemoveAll(mapPath)
	if err != nil {
		return fmt.Errorf("iscsi: failed to delete the directory: %s\nError: %v", mapPath, err)
	}
	klog.V(4).Infof("iscsi: successfully detached disk: %s", mapPath)
	return nil
}


func (c *iscsiDiskUnmapper) UnmapPodDevice() error {
	return nil
}


// GetGlobalMapPath returns global map path and error
// path: plugins/kubernetes.io/{PluginName}/volumeDevices/{ifaceName}/{portal-some_iqn-lun-lun_id}
func (iscsi *iscsiDisk) GetGlobalMapPath(spec *volume.Spec) (string, error) {
	return iscsi.iscsiGlobalMapPath(spec)
}


// GetPodDeviceMapPath returns pod device map path and volume name
// path: pods/{podUid}/volumeDevices/kubernetes.io~iscsi
// volumeName: pv0001
func (iscsi *iscsiDisk) GetPodDeviceMapPath() (string, string) {
	return iscsi.iscsiPodDeviceMapPath()
}


func portalMounter(portal string) string {
	if !strings.Contains(portal, ":") {
		portal = portal + ":3260"
	}
	return portal
}


// get iSCSI volume info: readOnly and fstype
func getISCSIVolumeInfo(spec *volume.Spec) (bool, string, error) {
	// for volume source, readonly is in volume spec
	// for PV, readonly is in PV spec. PV gets the ReadOnly flag indirectly through the PVC source
	if spec.Volume != nil && spec.Volume.ISCSI != nil {
		return spec.Volume.ISCSI.ReadOnly, spec.Volume.ISCSI.FSType, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.ISCSI != nil {
		return spec.ReadOnly, spec.PersistentVolume.Spec.ISCSI.FSType, nil
	}

	return false, "", fmt.Errorf("Spec does not reference an ISCSI volume type")
}


// get iSCSI target info: target portal, portals, iqn, and lun
func getISCSITargetInfo(spec *volume.Spec) (string, []string, string, int32, error) {
	if spec.Volume != nil && spec.Volume.ISCSI != nil {
		return spec.Volume.ISCSI.TargetPortal, spec.Volume.ISCSI.Portals, spec.Volume.ISCSI.IQN, spec.Volume.ISCSI.Lun, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.ISCSI != nil {
		return spec.PersistentVolume.Spec.ISCSI.TargetPortal, spec.PersistentVolume.Spec.ISCSI.Portals, spec.PersistentVolume.Spec.ISCSI.IQN, spec.PersistentVolume.Spec.ISCSI.Lun, nil
	}

	return "", nil, "", 0, fmt.Errorf("Spec does not reference an ISCSI volume type")
}


// get iSCSI initiator info: iface and initiator name
func getISCSIInitiatorInfo(spec *volume.Spec) (string, *string, error) {
	if spec.Volume != nil && spec.Volume.ISCSI != nil {
		return spec.Volume.ISCSI.ISCSIInterface, spec.Volume.ISCSI.InitiatorName, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.ISCSI != nil {
		return spec.PersistentVolume.Spec.ISCSI.ISCSIInterface, spec.PersistentVolume.Spec.ISCSI.InitiatorName, nil
	}

	return "", nil, fmt.Errorf("Spec does not reference an ISCSI volume type")
}


// get iSCSI Discovery CHAP boolean
func getISCSIDiscoveryCHAPInfo(spec *volume.Spec) (bool, error) {
	if spec.Volume != nil && spec.Volume.ISCSI != nil {
		return spec.Volume.ISCSI.DiscoveryCHAPAuth, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.ISCSI != nil {
		return spec.PersistentVolume.Spec.ISCSI.DiscoveryCHAPAuth, nil
	}

	return false, fmt.Errorf("Spec does not reference an ISCSI volume type")
}


// get iSCSI Session CHAP boolean
func getISCSISessionCHAPInfo(spec *volume.Spec) (bool, error) {
	if spec.Volume != nil && spec.Volume.ISCSI != nil {
		return spec.Volume.ISCSI.SessionCHAPAuth, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.ISCSI != nil {
		return spec.PersistentVolume.Spec.ISCSI.SessionCHAPAuth, nil
	}

	return false, fmt.Errorf("Spec does not reference an ISCSI volume type")
}


// get iSCSI CHAP Secret info: secret name and namespace
func getISCSISecretNameAndNamespace(spec *volume.Spec, defaultSecretNamespace string) (string, string, error) {
	if spec.Volume != nil && spec.Volume.ISCSI != nil {
		if spec.Volume.ISCSI.SecretRef != nil {
			return spec.Volume.ISCSI.SecretRef.Name, defaultSecretNamespace, nil
		}
		return "", "", nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.ISCSI != nil {
		secretRef := spec.PersistentVolume.Spec.ISCSI.SecretRef
		secretNs := defaultSecretNamespace
		if secretRef != nil {
			if len(secretRef.Namespace) != 0 {
				secretNs = secretRef.Namespace
			}
			return secretRef.Name, secretNs, nil
		}
		return "", "", nil
	}

	return "", "", fmt.Errorf("Spec does not reference an ISCSI volume type")
}