CreateVolume

// CreateVolume creates a GCE PD.
// Returns: gcePDName, volumeSizeGB, labels, fsType, error
func (util *GCEDiskUtil) CreateVolume(c *gcePersistentDiskProvisioner, node *v1.Node, allowedTopologies []v1.TopologySelectorTerm) (string, int, map[string]string, string, error) {
	cloud, err := getCloudProvider(c.gcePersistentDisk.plugin.host.GetCloudProvider())
	if err != nil {
		return "", 0, nil, "", err
	}

	name := volumeutil.GenerateVolumeName(c.options.ClusterName, c.options.PVName, 63) // GCE PD name can have up to 63 characters
	capacity := c.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	// GCE PDs are allocated in chunks of GiBs
	requestGB, err := volumehelpers.RoundUpToGiB(capacity)
	if err != nil {
		return "", 0, nil, "", err
	}

	// Apply Parameters.
	// Values for parameter "replication-type" are canonicalized to lower case.
	// Values for other parameters are case-insensitive, and we leave validation of these values
	// to the cloud provider.
	diskType := ""
	configuredZone := ""
	var configuredZones sets.String
	zonePresent := false
	zonesPresent := false
	replicationType := replicationTypeNone
	fstype := ""
	for k, v := range c.options.Parameters {
		switch strings.ToLower(k) {
		case "type":
			diskType = v
		case "zone":
			zonePresent = true
			configuredZone = v
		case "zones":
			zonesPresent = true
			configuredZones, err = volumehelpers.ZonesToSet(v)
			if err != nil {
				return "", 0, nil, "", err
			}
		case "replication-type":
			replicationType = strings.ToLower(v)
		case volume.VolumeParameterFSType:
			fstype = v
		default:
			return "", 0, nil, "", fmt.Errorf("invalid option %q for volume plugin %s", k, c.plugin.GetPluginName())
		}
	}

	// TODO: implement PVC.Selector parsing
	if c.options.PVC.Spec.Selector != nil {
		return "", 0, nil, "", fmt.Errorf("claim.Spec.Selector is not supported for dynamic provisioning on GCE")
	}

	var activezones sets.String
	activezones, err = cloud.GetAllCurrentZones()
	if err != nil {
		return "", 0, nil, "", err
	}

	var disk *gcecloud.Disk
	switch replicationType {
	case replicationTypeRegionalPD:
		selectedZones, err := volumehelpers.SelectZonesForVolume(zonePresent, zonesPresent, configuredZone, configuredZones, activezones, node, allowedTopologies, c.options.PVC.Name, maxRegionalPDZones)
		if err != nil {
			klog.V(2).Infof("Error selecting zones for regional GCE PD volume: %v", err)
			return "", 0, nil, "", err
		}
		disk, err = cloud.CreateRegionalDisk(
			name,
			diskType,
			selectedZones,
			requestGB,
			*c.options.CloudTags)
		if err != nil {
			klog.V(2).Infof("Error creating regional GCE PD volume: %v", err)
			return "", 0, nil, "", err
		}
		klog.V(2).Infof("Successfully created Regional GCE PD volume %s", name)

	case replicationTypeNone:
		selectedZone, err := volumehelpers.SelectZoneForVolume(zonePresent, zonesPresent, configuredZone, configuredZones, activezones, node, allowedTopologies, c.options.PVC.Name)
		if err != nil {
			return "", 0, nil, "", err
		}
		disk, err = cloud.CreateDisk(
			name,
			diskType,
			selectedZone,
			requestGB,
			*c.options.CloudTags)
		if err != nil {
			klog.V(2).Infof("Error creating single-zone GCE PD volume: %v", err)
			return "", 0, nil, "", err
		}
		klog.V(2).Infof("Successfully created single-zone GCE PD volume %s", name)

	default:
		return "", 0, nil, "", fmt.Errorf("replication-type of '%s' is not supported", replicationType)
	}

	labels, err := cloud.GetAutoLabelsForPD(disk)
	if err != nil {
		// We don't really want to leak the volume here...
		klog.Errorf("error getting labels for volume %q: %v", name, err)
	}

	return name, int(requestGB), labels, fstype, nil
}


verifyDevicePath

// Returns the first path that exists, or empty string if none exist.
func verifyDevicePath(devicePaths []string, sdBeforeSet sets.String, diskName string) (string, error) {
	if err := udevadmChangeToNewDrives(sdBeforeSet); err != nil {
		// It's possible udevadm was called on other disks so it should not block this
		// call. If it did fail on this disk, then the devicePath will either
		// not exist or be wrong. If it's wrong, then the scsi_id check below will fail.
		klog.Errorf("udevadmChangeToNewDrives failed with: %v", err)
	}

	for _, path := range devicePaths {
		if pathExists, err := mount.PathExists(path); err != nil {
			return "", fmt.Errorf("error checking if path exists: %v", err)
		} else if pathExists {
			// validate that the path actually resolves to the correct disk
			serial, err := getScsiSerial(path, diskName)
			if err != nil {
				return "", fmt.Errorf("failed to get scsi serial %v", err)
			}
			if serial != diskName {
				// The device link is not pointing to the correct device
				// Trigger udev on this device to try to fix the link
				if udevErr := udevadmChangeToDrive(path); udevErr != nil {
					klog.Errorf("udevadmChangeToDrive %q failed with: %v", path, err)
				}

				// Return error to retry WaitForAttach and verifyDevicePath
				return "", fmt.Errorf("scsi_id serial %q for device %q doesn't match disk %q", serial, path, diskName)
			}
			// The device link is correct
			return path, nil
		}
	}

	return "", nil
}


getScsiSerial

// Calls scsi_id on the given devicePath to get the serial number reported by that device.
func getScsiSerial(devicePath, diskName string) (string, error) {
	exists, err := utilpath.Exists(utilpath.CheckFollowSymlink, "/lib/udev/scsi_id")
	if err != nil {
		return "", fmt.Errorf("failed to check scsi_id existence: %v", err)
	}

	if !exists {
		klog.V(6).Infof("scsi_id doesn't exist; skipping check for %v", devicePath)
		return diskName, nil
	}

	out, err := exec.New().Command(
		"/lib/udev/scsi_id",
		"--page=0x83",
		"--whitelisted",
		fmt.Sprintf("--device=%v", devicePath)).CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("scsi_id failed for device %q with %v", devicePath, err)
	}

	return parseScsiSerial(string(out))
}


parseScsiSerial

// Parse the output returned by scsi_id and extract the serial number
func parseScsiSerial(output string) (string, error) {
	substrings := scsiRegex.FindStringSubmatch(output)
	if substrings == nil {
		return "", fmt.Errorf("scsi_id output cannot be parsed: %q", output)
	}

	return substrings[1], nil
}


getDiskByIDPaths

// Returns list of all /dev/disk/by-id/* paths for given PD.
func getDiskByIDPaths(pdName string, partition string) []string {
	devicePaths := []string{
		filepath.Join(diskByIDPath, diskGooglePrefix+pdName),
		filepath.Join(diskByIDPath, diskScsiGooglePrefix+pdName),
	}

	if partition != "" {
		for i, path := range devicePaths {
			devicePaths[i] = path + diskPartitionSuffix + partition
		}
	}

	return devicePaths
}


getCloudProvider

// Return cloud provider
func getCloudProvider(cloudProvider cloudprovider.Interface) (*gcecloud.Cloud, error) {
	var err error
	for numRetries := 0; numRetries < maxRetries; numRetries++ {
		gceCloudProvider, ok := cloudProvider.(*gcecloud.Cloud)
		if !ok || gceCloudProvider == nil {
			// Retry on error. See issue #11321
			klog.Errorf("Failed to get GCE Cloud Provider. plugin.host.GetCloudProvider returned %v instead", cloudProvider)
			time.Sleep(errorSleepDuration)
			continue
		}

		return gceCloudProvider, nil
	}

	return nil, fmt.Errorf("failed to get GCE GCECloudProvider with error %v", err)
}


udevadmChangeToNewDrives

// Triggers the application of udev rules by calling "udevadm trigger
// --action=change" for newly created "/dev/sd*" drives (exist only in
// after set). This is workaround for Issue #7972. Once the underlying
// issue has been resolved, this may be removed.
func udevadmChangeToNewDrives(sdBeforeSet sets.String) error {
	sdAfter, err := filepath.Glob(diskSDPattern)
	if err != nil {
		return fmt.Errorf("error filepath.Glob(\"%s\"): %v\r", diskSDPattern, err)
	}

	for _, sd := range sdAfter {
		if !sdBeforeSet.Has(sd) {
			return udevadmChangeToDrive(sd)
		}
	}

	return nil
}


udevadmChangeToDrive

// Calls "udevadm trigger --action=change" on the specified drive.
// drivePath must be the block device path to trigger on, in the format "/dev/sd*", or a symlink to it.
// This is workaround for Issue #7972. Once the underlying issue has been resolved, this may be removed.
func udevadmChangeToDrive(drivePath string) error {
	klog.V(5).Infof("udevadmChangeToDrive: drive=%q", drivePath)

	// Evaluate symlink, if any
	drive, err := filepath.EvalSymlinks(drivePath)
	if err != nil {
		return fmt.Errorf("udevadmChangeToDrive: filepath.EvalSymlinks(%q) failed with %v", drivePath, err)
	}
	klog.V(5).Infof("udevadmChangeToDrive: symlink path is %q", drive)

	// Check to make sure input is "/dev/sd*"
	if !strings.Contains(drive, diskSDPath) {
		return fmt.Errorf("udevadmChangeToDrive: expected input in the form \"%s\" but drive is %q", diskSDPattern, drive)
	}

	// Call "udevadm trigger --action=change --property-match=DEVNAME=/dev/sd..."
	_, err = exec.New().Command(
		"udevadm",
		"trigger",
		"--action=change",
		fmt.Sprintf("--property-match=DEVNAME=%s", drive)).CombinedOutput()
	if err != nil {
		return fmt.Errorf("udevadmChangeToDrive: udevadm trigger failed for drive %q with %v", drive, err)
	}
	return nil
}


isRegionalPD

// Checks whether the given GCE PD volume spec is associated with a regional PD.
func isRegionalPD(spec *volume.Spec) bool {
	if spec.PersistentVolume != nil {
		zonesLabel := spec.PersistentVolume.Labels[v1.LabelTopologyZone]
		if zonesLabel == "" {
			zonesLabel = spec.PersistentVolume.Labels[v1.LabelFailureDomainBetaZone]
		}
		zones := strings.Split(zonesLabel, cloudvolume.LabelMultiZoneDelimiter)
		return len(zones) > 1
	}
	return false
}


ProbeVolumePlugins

// This is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&gitRepoPlugin{nil}}
}


wrappedVolumeSpec

func wrappedVolumeSpec() volume.Spec {
	return volume.Spec{
		Volume: &v1.Volume{VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}},
	}
}


Init

func (plugin *gitRepoPlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	return nil
}


GetPluginName

func (plugin *gitRepoPlugin) GetPluginName() string {
	return gitRepoPluginName
}


GetVolumeName

func (plugin *gitRepoPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	volumeSource, _ := getVolumeSource(spec)
	if volumeSource == nil {
		return "", fmt.Errorf("Spec does not reference a Git repo volume type")
	}

	return fmt.Sprintf(
		"%v:%v:%v",
		volumeSource.Repository,
		volumeSource.Revision,
		volumeSource.Directory), nil
}


CanSupport

func (plugin *gitRepoPlugin) CanSupport(spec *volume.Spec) bool {
	return spec.Volume != nil && spec.Volume.GitRepo != nil
}


RequiresRemount

func (plugin *gitRepoPlugin) RequiresRemount(spec *volume.Spec) bool {
	return false
}


SupportsMountOption

func (plugin *gitRepoPlugin) SupportsMountOption() bool {
	return false
}


SupportsBulkVolumeVerification

func (plugin *gitRepoPlugin) SupportsBulkVolumeVerification() bool {
	return false
}


SupportsSELinuxContextMount

func (plugin *gitRepoPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return false, nil
}


NewMounter

func (plugin *gitRepoPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) {
	if err := validateVolume(spec.Volume.GitRepo); err != nil {
		return nil, err
	}

	return &gitRepoVolumeMounter{
		gitRepoVolume: &gitRepoVolume{
			volName: spec.Name(),
			podUID:  pod.UID,
			plugin:  plugin,
		},
		pod:      *pod,
		source:   spec.Volume.GitRepo.Repository,
		revision: spec.Volume.GitRepo.Revision,
		target:   spec.Volume.GitRepo.Directory,
		exec:     exec.New(),
		opts:     opts,
	}, nil
}


NewUnmounter

func (plugin *gitRepoPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	return &gitRepoVolumeUnmounter{
		&gitRepoVolume{
			volName: volName,
			podUID:  podUID,
			plugin:  plugin,
		},
	}, nil
}


ConstructVolumeSpec

func (plugin *gitRepoPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
	gitVolume := &v1.Volume{
		Name: volumeName,
		VolumeSource: v1.VolumeSource{
			GitRepo: &v1.GitRepoVolumeSource{},
		},
	}
	return volume.ReconstructedVolume{
		Spec: volume.NewSpecFromVolume(gitVolume),
	}, nil
}


GetPath

func (gr *gitRepoVolume) GetPath() string {
	name := gitRepoPluginName
	return gr.plugin.host.GetPodVolumeDir(gr.podUID, utilstrings.EscapeQualifiedName(name), gr.volName)
}


GetAttributes

func (b *gitRepoVolumeMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:       false,
		Managed:        true,
		SELinuxRelabel: true, // xattr change should be okay, TODO: double check
	}
}


SetUp

// SetUp creates new directory and clones a git repo.
func (b *gitRepoVolumeMounter) SetUp(mounterArgs volume.MounterArgs) error {
	return b.SetUpAt(b.GetPath(), mounterArgs)
}


SetUpAt

// SetUpAt creates new directory and clones a git repo.
func (b *gitRepoVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	if volumeutil.IsReady(b.getMetaDir()) {
		return nil
	}

	// Wrap EmptyDir, let it do the setup.
	wrapped, err := b.plugin.host.NewWrapperMounter(b.volName, wrappedVolumeSpec(), &b.pod, b.opts)
	if err != nil {
		return err
	}
	if err := wrapped.SetUpAt(dir, mounterArgs); err != nil {
		return err
	}

	args := []string{"clone", "--", b.source}

	if len(b.target) != 0 {
		args = append(args, b.target)
	}
	if output, err := b.execCommand("git", args, dir); err != nil {
		return fmt.Errorf("failed to exec 'git %s': %s: %v",
			strings.Join(args, " "), output, err)
	}

	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return err
	}

	if len(b.revision) == 0 {
		// Done!
		volumeutil.SetReady(b.getMetaDir())
		return nil
	}

	var subdir string

	switch {
	case len(b.target) != 0 && filepath.Clean(b.target) == ".":
		// if target dir is '.', use the current dir
		subdir = filepath.Join(dir)
	case len(files) == 1:
		// if target is not '.', use the generated folder
		subdir = filepath.Join(dir, files[0].Name())
	default:
		// if target is not '.', but generated many files, it's wrong
		return fmt.Errorf("unexpected directory contents: %v", files)
	}

	if output, err := b.execCommand("git", []string{"checkout", b.revision}, subdir); err != nil {
		return fmt.Errorf("failed to exec 'git checkout %s': %s: %v", b.revision, output, err)
	}
	if output, err := b.execCommand("git", []string{"reset", "--hard"}, subdir); err != nil {
		return fmt.Errorf("failed to exec 'git reset --hard': %s: %v", output, err)
	}

	volume.SetVolumeOwnership(b, dir, mounterArgs.FsGroup, nil /*fsGroupChangePolicy*/, volumeutil.FSGroupCompleteHook(b.plugin, nil))

	volumeutil.SetReady(b.getMetaDir())
	return nil
}


getMetaDir

func (b *gitRepoVolumeMounter) getMetaDir() string {
	return filepath.Join(b.plugin.host.GetPodPluginDir(b.podUID, utilstrings.EscapeQualifiedName(gitRepoPluginName)), b.volName)
}


execCommand

func (b *gitRepoVolumeMounter) execCommand(command string, args []string, dir string) ([]byte, error) {
	cmd := b.exec.Command(command, args...)
	cmd.SetDir(dir)
	return cmd.CombinedOutput()
}


validateVolume

func validateVolume(src *v1.GitRepoVolumeSource) error {
	if err := validateNonFlagArgument(src.Repository, "repository"); err != nil {
		return err
	}
	if err := validateNonFlagArgument(src.Revision, "revision"); err != nil {
		return err
	}
	if err := validateNonFlagArgument(src.Directory, "directory"); err != nil {
		return err
	}
	return nil
}


TearDown

// TearDown simply deletes everything in the directory.
func (c *gitRepoVolumeUnmounter) TearDown() error {
	return c.TearDownAt(c.GetPath())
}


TearDownAt

// TearDownAt simply deletes everything in the directory.
func (c *gitRepoVolumeUnmounter) TearDownAt(dir string) error {
	return volumeutil.UnmountViaEmptyDir(dir, c.plugin.host, c.volName, wrappedVolumeSpec(), c.podUID)
}


getVolumeSource

func getVolumeSource(spec *volume.Spec) (*v1.GitRepoVolumeSource, bool) {
	var readOnly bool
	var volumeSource *v1.GitRepoVolumeSource

	if spec.Volume != nil && spec.Volume.GitRepo != nil {
		volumeSource = spec.Volume.GitRepo
		readOnly = spec.ReadOnly
	}

	return volumeSource, readOnly
}


validateNonFlagArgument

func validateNonFlagArgument(arg, argName string) error {
	if len(arg) > 0 && arg[0] == '-' {
		return fmt.Errorf("%q is an invalid value for %s", arg, argName)
	}
	return nil
}


ProbeVolumePlugins

// ProbeVolumePlugins is the primary entrypoint for volume plugins.
// The volumeConfig arg provides the ability to configure volume behavior.  It is implemented as a pointer to allow nils.
// The hostPathPlugin is used to store the volumeConfig and give it, when needed, to the func that Recycles.
// Tests that exercise recycling should not use this func but instead use ProbeRecyclablePlugins() to override default behavior.
func ProbeVolumePlugins(volumeConfig volume.VolumeConfig) []volume.VolumePlugin {
	return []volume.VolumePlugin{
		&hostPathPlugin{
			host:   nil,
			config: volumeConfig,
		},
	}
}


FakeProbeVolumePlugins

func FakeProbeVolumePlugins(volumeConfig volume.VolumeConfig) []volume.VolumePlugin {
	return []volume.VolumePlugin{
		&hostPathPlugin{
			host:          nil,
			config:        volumeConfig,
			noTypeChecker: true,
		},
	}
}


Init

func (plugin *hostPathPlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	return nil
}


GetPluginName

func (plugin *hostPathPlugin) GetPluginName() string {
	return hostPathPluginName
}


GetVolumeName

func (plugin *hostPathPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return volumeSource.Path, nil
}


CanSupport

func (plugin *hostPathPlugin) CanSupport(spec *volume.Spec) bool {
	return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.HostPath != nil) ||
		(spec.Volume != nil && spec.Volume.HostPath != nil)
}


RequiresRemount

func (plugin *hostPathPlugin) RequiresRemount(spec *volume.Spec) bool {
	return false
}


SupportsMountOption

func (plugin *hostPathPlugin) SupportsMountOption() bool {
	return false
}


SupportsBulkVolumeVerification

func (plugin *hostPathPlugin) SupportsBulkVolumeVerification() bool {
	return false
}


SupportsSELinuxContextMount

func (plugin *hostPathPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return false, nil
}


GetAccessModes

func (plugin *hostPathPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
	return []v1.PersistentVolumeAccessMode{
		v1.ReadWriteOnce,
	}
}


NewMounter

func (plugin *hostPathPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) {
	hostPathVolumeSource, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}

	path := hostPathVolumeSource.Path
	pathType := new(v1.HostPathType)
	if hostPathVolumeSource.Type == nil {
		*pathType = v1.HostPathUnset
	} else {
		pathType = hostPathVolumeSource.Type
	}
	kvh, ok := plugin.host.(volume.KubeletVolumeHost)
	if !ok {
		return nil, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
	}
	return &hostPathMounter{
		hostPath:      &hostPath{path: path, pathType: pathType},
		readOnly:      readOnly,
		mounter:       plugin.host.GetMounter(plugin.GetPluginName()),
		hu:            kvh.GetHostUtil(),
		noTypeChecker: plugin.noTypeChecker,
	}, nil
}


NewUnmounter

func (plugin *hostPathPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	return &hostPathUnmounter{&hostPath{
		path: "",
	}}, nil
}


Recycle

// Recycle recycles/scrubs clean a HostPath volume.
// Recycle blocks until the pod has completed or any error occurs.
// HostPath recycling only works in single node clusters and is meant for testing purposes only.
func (plugin *hostPathPlugin) Recycle(pvName string, spec *volume.Spec, eventRecorder recyclerclient.RecycleEventRecorder) error {
	if spec.PersistentVolume == nil || spec.PersistentVolume.Spec.HostPath == nil {
		return fmt.Errorf("spec.PersistentVolume.Spec.HostPath is nil")
	}

	pod := plugin.config.RecyclerPodTemplate
	timeout := util.CalculateTimeoutForVolume(plugin.config.RecyclerMinimumTimeout, plugin.config.RecyclerTimeoutIncrement, spec.PersistentVolume)
	// overrides
	pod.Spec.ActiveDeadlineSeconds = &timeout
	pod.Spec.Volumes[0].VolumeSource = v1.VolumeSource{
		HostPath: &v1.HostPathVolumeSource{
			Path: spec.PersistentVolume.Spec.HostPath.Path,
		},
	}
	return recyclerclient.RecycleVolumeByWatchingPodUntilCompletion(pvName, pod, plugin.host.GetKubeClient(), eventRecorder)
}


NewDeleter

func (plugin *hostPathPlugin) NewDeleter(logger klog.Logger, spec *volume.Spec) (volume.Deleter, error) {
	return newDeleter(spec, plugin.host)
}


NewProvisioner

func (plugin *hostPathPlugin) NewProvisioner(logger klog.Logger, options volume.VolumeOptions) (volume.Provisioner, error) {
	if !plugin.config.ProvisioningEnabled {
		return nil, fmt.Errorf("provisioning in volume plugin %q is disabled", plugin.GetPluginName())
	}
	return newProvisioner(options, plugin.host, plugin)
}


ConstructVolumeSpec

func (plugin *hostPathPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
	hostPathVolume := &v1.Volume{
		Name: volumeName,
		VolumeSource: v1.VolumeSource{
			HostPath: &v1.HostPathVolumeSource{
				Path: volumeName,
			},
		},
	}
	return volume.ReconstructedVolume{
		Spec: volume.NewSpecFromVolume(hostPathVolume),
	}, nil
}


newDeleter

func newDeleter(spec *volume.Spec, host volume.VolumeHost) (volume.Deleter, error) {
	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.HostPath == nil {
		return nil, fmt.Errorf("spec.PersistentVolumeSource.HostPath is nil")
	}
	path := spec.PersistentVolume.Spec.HostPath.Path
	return &hostPathDeleter{name: spec.Name(), path: path, host: host}, nil
}


newProvisioner

func newProvisioner(options volume.VolumeOptions, host volume.VolumeHost, plugin *hostPathPlugin) (volume.Provisioner, error) {
	return &hostPathProvisioner{options: options, host: host, plugin: plugin, basePath: "hostpath_pv"}, nil
}


GetPath

func (hp *hostPath) GetPath() string {
	return hp.path
}


GetAttributes

func (b *hostPathMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:       b.readOnly,
		Managed:        false,
		SELinuxRelabel: false,
	}
}


SetUp

// SetUp does nothing.
func (b *hostPathMounter) SetUp(mounterArgs volume.MounterArgs) error {
	err := validation.ValidatePathNoBacksteps(b.GetPath())
	if err != nil {
		return fmt.Errorf("invalid HostPath `%s`: %v", b.GetPath(), err)
	}

	if *b.pathType == v1.HostPathUnset {
		return nil
	}
	if b.noTypeChecker {
		return nil
	} else {
		return checkType(b.GetPath(), b.pathType, b.hu)
	}
}


SetUpAt

// SetUpAt does not make sense for host paths - probably programmer error.
func (b *hostPathMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	return fmt.Errorf("SetUpAt() does not make sense for host paths")
}


GetPath

func (b *hostPathMounter) GetPath() string {
	return b.path
}


TearDown

// TearDown does nothing.
func (c *hostPathUnmounter) TearDown() error {
	return nil
}


TearDownAt

// TearDownAt does not make sense for host paths - probably programmer error.
func (c *hostPathUnmounter) TearDownAt(dir string) error {
	return fmt.Errorf("TearDownAt() does not make sense for host paths")
}


Provision

// Create for hostPath simply creates a local /tmp/%/%s directory as a new PersistentVolume, default /tmp/hostpath_pv/%s.
// This Provisioner is meant for development and testing only and WILL NOT WORK in a multi-node cluster.
func (r *hostPathProvisioner) Provision(selectedNode *v1.Node, allowedTopologies []v1.TopologySelectorTerm) (*v1.PersistentVolume, error) {
	if util.CheckPersistentVolumeClaimModeBlock(r.options.PVC) {
		return nil, fmt.Errorf("%s does not support block volume provisioning", r.plugin.GetPluginName())
	}

	fullpath := fmt.Sprintf("/tmp/%s/%s", r.basePath, uuid.NewUUID())

	capacity := r.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	pv := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: r.options.PVName,
			Annotations: map[string]string{
				util.VolumeDynamicallyCreatedByKey: "hostpath-dynamic-provisioner",
			},
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeReclaimPolicy: r.options.PersistentVolumeReclaimPolicy,
			AccessModes:                   r.options.PVC.Spec.AccessModes,
			Capacity: v1.ResourceList{
				v1.ResourceName(v1.ResourceStorage): capacity,
			},
			PersistentVolumeSource: v1.PersistentVolumeSource{
				HostPath: &v1.HostPathVolumeSource{
					Path: fullpath,
				},
			},
		},
	}
	if len(r.options.PVC.Spec.AccessModes) == 0 {
		pv.Spec.AccessModes = r.plugin.GetAccessModes()
	}

	if err := os.MkdirAll(pv.Spec.HostPath.Path, 0750); err != nil {
		return nil, err
	}
	if selinux.GetEnabled() {
		err := selinux.SetFileLabel(pv.Spec.HostPath.Path, config.KubeletContainersSharedSELinuxLabel)
		if err != nil {
			return nil, fmt.Errorf("failed to set selinux label for %q: %v", pv.Spec.HostPath.Path, err)
		}
	}

	return pv, nil
}


GetPath

func (r *hostPathDeleter) GetPath() string {
	return r.path
}


Delete

// Delete for hostPath removes the local directory so long as it is beneath /tmp/*.
// THIS IS FOR TESTING AND LOCAL DEVELOPMENT ONLY!  This message should scare you away from using
// this deleter for anything other than development and testing.
func (r *hostPathDeleter) Delete() error {
	regexp := regexp.MustCompile("/tmp/.+")
	if !regexp.MatchString(r.GetPath()) {
		return fmt.Errorf("host_path deleter only supports /tmp/.+ but received provided %s", r.GetPath())
	}
	return os.RemoveAll(r.GetPath())
}


getVolumeSource

func getVolumeSource(spec *volume.Spec) (*v1.HostPathVolumeSource, bool, error) {
	if spec.Volume != nil && spec.Volume.HostPath != nil {
		return spec.Volume.HostPath, spec.ReadOnly, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.HostPath != nil {
		return spec.PersistentVolume.Spec.HostPath, spec.ReadOnly, nil
	}

	return nil, false, fmt.Errorf("spec does not reference an HostPath volume type")
}


Exists

func (ftc *fileTypeChecker) Exists() bool {
	exists, err := ftc.hu.PathExists(ftc.path)
	return exists && err == nil
}


IsFile

func (ftc *fileTypeChecker) IsFile() bool {
	if !ftc.Exists() {
		return false
	}
	pathType, err := ftc.hu.GetFileType(ftc.path)
	if err != nil {
		return false
	}
	return string(pathType) == string(v1.HostPathFile)
}


MakeFile

func (ftc *fileTypeChecker) MakeFile() error {
	return makeFile(ftc.path)
}


IsDir

func (ftc *fileTypeChecker) IsDir() bool {
	if !ftc.Exists() {
		return false
	}
	pathType, err := ftc.hu.GetFileType(ftc.path)
	if err != nil {
		return false
	}
	return string(pathType) == string(v1.HostPathDirectory)
}


MakeDir

func (ftc *fileTypeChecker) MakeDir() error {
	return makeDir(ftc.path)
}


IsBlock

func (ftc *fileTypeChecker) IsBlock() bool {
	blkDevType, err := ftc.hu.GetFileType(ftc.path)
	if err != nil {
		return false
	}
	return string(blkDevType) == string(v1.HostPathBlockDev)
}


IsChar

func (ftc *fileTypeChecker) IsChar() bool {
	charDevType, err := ftc.hu.GetFileType(ftc.path)
	if err != nil {
		return false
	}
	return string(charDevType) == string(v1.HostPathCharDev)
}


IsSocket

func (ftc *fileTypeChecker) IsSocket() bool {
	socketType, err := ftc.hu.GetFileType(ftc.path)
	if err != nil {
		return false
	}
	return string(socketType) == string(v1.HostPathSocket)
}


GetPath

func (ftc *fileTypeChecker) GetPath() string {
	return ftc.path
}


newFileTypeChecker

func newFileTypeChecker(path string, hu hostutil.HostUtils) hostPathTypeChecker {
	return &fileTypeChecker{path: path, hu: hu}
}


checkType

// checkType checks whether the given path is the exact pathType
func checkType(path string, pathType *v1.HostPathType, hu hostutil.HostUtils) error {
	return checkTypeInternal(newFileTypeChecker(path, hu), pathType)
}


checkTypeInternal

func checkTypeInternal(ftc hostPathTypeChecker, pathType *v1.HostPathType) error {
	switch *pathType {
	case v1.HostPathDirectoryOrCreate:
		if !ftc.Exists() {
			return ftc.MakeDir()
		}
		fallthrough
	case v1.HostPathDirectory:
		if !ftc.IsDir() {
			return fmt.Errorf("hostPath type check failed: %s is not a directory", ftc.GetPath())
		}
	case v1.HostPathFileOrCreate:
		if !ftc.Exists() {
			return ftc.MakeFile()
		}
		fallthrough
	case v1.HostPathFile:
		if !ftc.IsFile() {
			return fmt.Errorf("hostPath type check failed: %s is not a file", ftc.GetPath())
		}
	case v1.HostPathSocket:
		if !ftc.IsSocket() {
			return fmt.Errorf("hostPath type check failed: %s is not a socket file", ftc.GetPath())
		}
	case v1.HostPathCharDev:
		if !ftc.IsChar() {
			return fmt.Errorf("hostPath type check failed: %s is not a character device", ftc.GetPath())
		}
	case v1.HostPathBlockDev:
		if !ftc.IsBlock() {
			return fmt.Errorf("hostPath type check failed: %s is not a block device", ftc.GetPath())
		}
	default:
		return fmt.Errorf("%s is an invalid volume type", *pathType)
	}

	return nil
}


makeDir

// makeDir creates a new directory.
// If pathname already exists as a directory, no error is returned.
// If pathname already exists as a file, an error is returned.
func makeDir(pathname string) error {
	err := os.MkdirAll(pathname, os.FileMode(0755))
	if err != nil {
		if !os.IsExist(err) {
			return err
		}
	}
	return nil
}


makeFile

// makeFile creates an empty file.
// If pathname already exists, whether a file or directory, no error is returned.
func makeFile(pathname string) error {
	f, err := os.OpenFile(pathname, os.O_CREATE, os.FileMode(0644))
	if f != nil {
		f.Close()
	}
	if err != nil {
		if !os.IsExist(err) {
			return err
		}
	}
	return nil
}


NewAttacher

func (plugin *iscsiPlugin) NewAttacher() (volume.Attacher, error) {
	return &iscsiAttacher{
		host:        plugin.host,
		targetLocks: plugin.targetLocks,
		manager:     &ISCSIUtil{},
	}, nil
}


NewDeviceMounter

func (plugin *iscsiPlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
	return plugin.NewAttacher()
}


GetDeviceMountRefs

func (plugin *iscsiPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
	mounter := plugin.host.GetMounter(iscsiPluginName)
	return mounter.GetMountRefs(deviceMountPath)
}


Attach

func (attacher *iscsiAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) {
	return "", nil
}


VolumesAreAttached

func (attacher *iscsiAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.NodeName) (map[*volume.Spec]bool, error) {
	volumesAttachedCheck := make(map[*volume.Spec]bool)
	for _, spec := range specs {
		volumesAttachedCheck[spec] = true
	}

	return volumesAttachedCheck, nil
}


WaitForAttach

func (attacher *iscsiAttacher) WaitForAttach(spec *volume.Spec, devicePath string, pod *v1.Pod, timeout time.Duration) (string, error) {
	mounter, err := volumeSpecToMounter(spec, attacher.host, attacher.targetLocks, pod)
	if err != nil {
		klog.Warningf("failed to get iscsi mounter: %v", err)
		return "", err
	}
	return attacher.manager.AttachDisk(*mounter)
}


GetDeviceMountPath

func (attacher *iscsiAttacher) GetDeviceMountPath(
	spec *volume.Spec) (string, error) {
	mounter, err := volumeSpecToMounter(spec, attacher.host, attacher.targetLocks, nil)
	if err != nil {
		klog.Warningf("failed to get iscsi mounter: %v", err)
		return "", err
	}
	if mounter.InitiatorName != "" {
		// new iface name is <target portal>:<volume name>
		mounter.Iface = mounter.Portals[0] + ":" + mounter.VolName
	}
	return attacher.manager.MakeGlobalPDName(*mounter.iscsiDisk), nil
}


MountDevice

func (attacher *iscsiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string, mountArgs volume.DeviceMounterArgs) error {
	mounter := attacher.host.GetMounter(iscsiPluginName)
	notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath)
	if err != nil {
		if os.IsNotExist(err) {
			if err := os.MkdirAll(deviceMountPath, 0750); err != nil {
				return err
			}
			notMnt = true
		} else {
			return err
		}
	}
	readOnly, fsType, err := getISCSIVolumeInfo(spec)
	if err != nil {
		return err
	}

	options := []string{}
	if readOnly {
		options = append(options, "ro")
	}
	if mountArgs.SELinuxLabel != "" {
		options = volumeutil.AddSELinuxMountOption(options, mountArgs.SELinuxLabel)
	}
	if notMnt {
		diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Exec: attacher.host.GetExec(iscsiPluginName)}
		mountOptions := volumeutil.MountOptionFromSpec(spec, options...)
		err = diskMounter.FormatAndMount(devicePath, deviceMountPath, fsType, mountOptions)
		if err != nil {
			os.Remove(deviceMountPath)
			return err
		}
	}
	return nil
}


NewDetacher

func (plugin *iscsiPlugin) NewDetacher() (volume.Detacher, error) {
	return &iscsiDetacher{
		host:    plugin.host,
		mounter: plugin.host.GetMounter(iscsiPluginName),
		manager: &ISCSIUtil{},
		plugin:  plugin,
	}, nil
}


NewDeviceUnmounter

func (plugin *iscsiPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
	return plugin.NewDetacher()
}


Detach

func (detacher *iscsiDetacher) Detach(volumeName string, nodeName types.NodeName) error {
	return nil
}


UnmountDevice

func (detacher *iscsiDetacher) UnmountDevice(deviceMountPath string) error {
	unMounter := volumeSpecToUnmounter(detacher.mounter, detacher.host, detacher.plugin)
	err := detacher.manager.DetachDisk(*unMounter, deviceMountPath)
	if err != nil {
		return fmt.Errorf("iscsi: failed to detach disk: %s\nError: %v", deviceMountPath, err)
	}
	klog.V(4).Infof("iscsi: %q is unmounted, deleting the directory", deviceMountPath)
	err = os.RemoveAll(deviceMountPath)
	if err != nil {
		return fmt.Errorf("iscsi: failed to delete the directory: %s\nError: %v", deviceMountPath, err)
	}
	klog.V(4).Infof("iscsi: successfully detached disk: %s", deviceMountPath)
	return nil
}


CanAttach

func (plugin *iscsiPlugin) CanAttach(spec *volume.Spec) (bool, error) {
	return true, nil
}


CanDeviceMount

func (plugin *iscsiPlugin) CanDeviceMount(spec *volume.Spec) (bool, error) {
	return true, nil
}


volumeSpecToMounter

func volumeSpecToMounter(spec *volume.Spec, host volume.VolumeHost, targetLocks keymutex.KeyMutex, pod *v1.Pod) (*iscsiDiskMounter, error) {
	var secret map[string]string
	readOnly, fsType, err := getISCSIVolumeInfo(spec)
	if err != nil {
		return nil, err
	}
	var podUID types.UID
	if pod != nil {
		secret, err = createSecretMap(spec, &iscsiPlugin{host: host, targetLocks: targetLocks}, pod.Namespace)
		if err != nil {
			return nil, err
		}
		podUID = pod.UID
	}
	iscsiDisk, err := createISCSIDisk(spec,
		podUID,
		&iscsiPlugin{host: host, targetLocks: targetLocks},
		&ISCSIUtil{},
		secret,
	)
	if err != nil {
		return nil, err
	}
	exec := host.GetExec(iscsiPluginName)

	volumeMode, err := volumeutil.GetVolumeMode(spec)
	if err != nil {
		return nil, err
	}

	klog.V(5).Infof("iscsi: VolumeSpecToMounter volumeMode %s", volumeMode)
	return &iscsiDiskMounter{
		iscsiDisk:  iscsiDisk,
		fsType:     fsType,
		volumeMode: volumeMode,
		readOnly:   readOnly,
		mounter:    &mount.SafeFormatAndMount{Interface: host.GetMounter(iscsiPluginName), Exec: exec},
		exec:       exec,
		deviceUtil: volumeutil.NewDeviceHandler(volumeutil.NewIOHandler()),
	}, nil
}


volumeSpecToUnmounter

func volumeSpecToUnmounter(mounter mount.Interface, host volume.VolumeHost, plugin *iscsiPlugin) *iscsiDiskUnmounter {
	exec := host.GetExec(iscsiPluginName)
	return &iscsiDiskUnmounter{
		iscsiDisk: &iscsiDisk{
			plugin: plugin,
		},
		mounter:    mounter,
		exec:       exec,
		deviceUtil: volumeutil.NewDeviceHandler(volumeutil.NewIOHandler()),
	}
}


diskSetUp

// utility to mount a disk based filesystem
// globalPDPath: global mount path like, /var/lib/kubelet/plugins/kubernetes.io/iscsi/{ifaceName}/{portal-some_iqn-lun-lun_id}
// volPath: pod volume dir path like, /var/lib/kubelet/pods/{podUID}/volumes/kubernetes.io~iscsi/{volumeName}
func diskSetUp(manager diskManager, b iscsiDiskMounter, volPath string, mounter mount.Interface, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy) error {
	notMnt, err := mounter.IsLikelyNotMountPoint(volPath)
	if err != nil && !os.IsNotExist(err) {
		klog.Errorf("cannot validate mountpoint: %s", volPath)
		return err
	}
	if !notMnt {
		return nil
	}

	if err := os.MkdirAll(volPath, 0750); err != nil {
		klog.Errorf("failed to mkdir:%s", volPath)
		return err
	}
	// Perform a bind mount to the full path to allow duplicate mounts of the same disk.
	options := []string{"bind"}
	if b.readOnly {
		options = append(options, "ro")
	}
	if b.iscsiDisk.InitiatorName != "" {
		// new iface name is <target portal>:<volume name>
		b.iscsiDisk.Iface = b.iscsiDisk.Portals[0] + ":" + b.iscsiDisk.VolName
	}
	globalPDPath := manager.MakeGlobalPDName(*b.iscsiDisk)
	mountOptions := util.JoinMountOptions(b.mountOptions, options)
	err = mounter.MountSensitiveWithoutSystemd(globalPDPath, volPath, "", mountOptions, nil)
	if err != nil {
		klog.Errorf("Failed to bind mount: source:%s, target:%s, err:%v", globalPDPath, volPath, err)
		noMnt, mntErr := b.mounter.IsLikelyNotMountPoint(volPath)
		if mntErr != nil {
			klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
			return err
		}
		if !noMnt {
			if mntErr = b.mounter.Unmount(volPath); mntErr != nil {
				klog.Errorf("Failed to unmount: %v", mntErr)
				return err
			}
			noMnt, mntErr = b.mounter.IsLikelyNotMountPoint(volPath)
			if mntErr != nil {
				klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
				return err
			}
			if !noMnt {
				//  will most likely retry on next sync loop.
				klog.Errorf("%s is still mounted, despite call to unmount().  Will try again next sync loop.", volPath)
				return err
			}
		}
		os.Remove(volPath)
		return err
	}

	if !b.readOnly {
		volume.SetVolumeOwnership(&b, volPath, fsGroup, fsGroupChangePolicy, util.FSGroupCompleteHook(b.plugin, nil))
	}

	return nil
}


ProbeVolumePlugins

// ProbeVolumePlugins is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&iscsiPlugin{}}
}


Init

func (plugin *iscsiPlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	plugin.targetLocks = keymutex.NewHashed(0)
	return nil
}


GetPluginName

func (plugin *iscsiPlugin) GetPluginName() string {
	return iscsiPluginName
}


GetVolumeName

func (plugin *iscsiPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	tp, _, iqn, lun, err := getISCSITargetInfo(spec)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%v:%v:%v", tp, iqn, lun), nil
}


CanSupport

func (plugin *iscsiPlugin) CanSupport(spec *volume.Spec) bool {
	return (spec.Volume != nil && spec.Volume.ISCSI != nil) || (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.ISCSI != nil)
}


RequiresRemount

func (plugin *iscsiPlugin) RequiresRemount(spec *volume.Spec) bool {
	return false
}


SupportsMountOption

func (plugin *iscsiPlugin) SupportsMountOption() bool {
	return true
}


SupportsBulkVolumeVerification

func (plugin *iscsiPlugin) SupportsBulkVolumeVerification() bool {
	return false
}


SupportsSELinuxContextMount

func (plugin *iscsiPlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return true, nil
}


GetAccessModes

func (plugin *iscsiPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
	return []v1.PersistentVolumeAccessMode{
		v1.ReadWriteOnce,
		v1.ReadOnlyMany,
	}
}


NewMounter

func (plugin *iscsiPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
	if pod == nil {
		return nil, fmt.Errorf("nil pod")
	}
	secret, err := createSecretMap(spec, plugin, pod.Namespace)
	if err != nil {
		return nil, err
	}
	return plugin.newMounterInternal(spec, pod.UID, &ISCSIUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()), secret)
}


newMounterInternal

func (plugin *iscsiPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface, exec utilexec.Interface, secret map[string]string) (volume.Mounter, error) {
	readOnly, fsType, err := getISCSIVolumeInfo(spec)
	if err != nil {
		return nil, err
	}
	iscsiDisk, err := createISCSIDisk(spec, podUID, plugin, manager, secret)
	if err != nil {
		return nil, err
	}

	if iscsiDisk != nil {

		//Add volume metrics
		iscsiDisk.MetricsProvider = volume.NewMetricsStatFS(iscsiDisk.GetPath())
	}
	return &iscsiDiskMounter{
		iscsiDisk:    iscsiDisk,
		fsType:       fsType,
		readOnly:     readOnly,
		mounter:      &mount.SafeFormatAndMount{Interface: mounter, Exec: exec},
		exec:         exec,
		deviceUtil:   ioutil.NewDeviceHandler(ioutil.NewIOHandler()),
		mountOptions: ioutil.MountOptionFromSpec(spec),
	}, nil
}


NewBlockVolumeMapper

// NewBlockVolumeMapper creates a new volume.BlockVolumeMapper from an API specification.
func (plugin *iscsiPlugin) NewBlockVolumeMapper(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.BlockVolumeMapper, error) {
	// If this is called via GenerateUnmapDeviceFunc(), pod is nil.
	// Pass empty string as dummy uid since uid isn't used in the case.
	var uid types.UID
	var secret map[string]string
	var err error
	if pod != nil {
		uid = pod.UID
		secret, err = createSecretMap(spec, plugin, pod.Namespace)
		if err != nil {
			return nil, err
		}
	}
	return plugin.newBlockVolumeMapperInternal(spec, uid, &ISCSIUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()), secret)
}


newBlockVolumeMapperInternal

func (plugin *iscsiPlugin) newBlockVolumeMapperInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface, exec utilexec.Interface, secret map[string]string) (volume.BlockVolumeMapper, error) {
	readOnly, _, err := getISCSIVolumeInfo(spec)
	if err != nil {
		return nil, err
	}
	iscsiDisk, err := createISCSIDisk(spec, podUID, plugin, manager, secret)
	if err != nil {
		return nil, err
	}
	mapper := &iscsiDiskMapper{
		iscsiDisk:  iscsiDisk,
		readOnly:   readOnly,
		exec:       exec,
		deviceUtil: ioutil.NewDeviceHandler(ioutil.NewIOHandler()),
	}

	blockPath, err := mapper.GetGlobalMapPath(spec)
	if err != nil {
		return nil, fmt.Errorf("failed to get device path: %v", err)
	}
	mapper.MetricsProvider = volume.NewMetricsBlock(filepath.Join(blockPath, string(podUID)))

	return mapper, nil
}


NewUnmounter

func (plugin *iscsiPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	return plugin.newUnmounterInternal(volName, podUID, &ISCSIUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()))
}


newUnmounterInternal

func (plugin *iscsiPlugin) newUnmounterInternal(volName string, podUID types.UID, manager diskManager, mounter mount.Interface, exec utilexec.Interface) (volume.Unmounter, error) {
	return &iscsiDiskUnmounter{
		iscsiDisk: &iscsiDisk{
			podUID:          podUID,
			VolName:         volName,
			manager:         manager,
			plugin:          plugin,
			MetricsProvider: volume.NewMetricsStatFS(plugin.host.GetPodVolumeDir(podUID, utilstrings.EscapeQualifiedName(iscsiPluginName), volName)),
		},
		mounter:    mounter,
		exec:       exec,
		deviceUtil: ioutil.NewDeviceHandler(ioutil.NewIOHandler()),
	}, nil
}


NewBlockVolumeUnmapper

// NewBlockVolumeUnmapper creates a new volume.BlockVolumeUnmapper from recoverable state.
func (plugin *iscsiPlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (volume.BlockVolumeUnmapper, error) {
	return plugin.newUnmapperInternal(volName, podUID, &ISCSIUtil{}, plugin.host.GetExec(plugin.GetPluginName()))
}


newUnmapperInternal

func (plugin *iscsiPlugin) newUnmapperInternal(volName string, podUID types.UID, manager diskManager, exec utilexec.Interface) (volume.BlockVolumeUnmapper, error) {
	return &iscsiDiskUnmapper{
		iscsiDisk: &iscsiDisk{
			podUID:  podUID,
			VolName: volName,
			manager: manager,
			plugin:  plugin,
		},
		exec:       exec,
		deviceUtil: ioutil.NewDeviceHandler(ioutil.NewIOHandler()),
	}, nil
}


ConstructVolumeSpec

func (plugin *iscsiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
	// Find globalPDPath from pod volume directory(mountPath)
	var globalPDPath string
	mounter := plugin.host.GetMounter(plugin.GetPluginName())
	// Try really hard to get the global mount of the volume, an error returned from here would
	// leave the global mount still mounted, while marking the volume as unused.
	// The volume can then be mounted on several nodes, resulting in volume
	// corruption.
	paths, err := ioutil.GetReliableMountRefs(mounter, mountPath)
	if io.IsInconsistentReadError(err) {
		klog.Errorf("Failed to read mount refs from /proc/mounts for %s: %s", mountPath, err)
		klog.Errorf("Kubelet cannot unmount volume at %s, please unmount it and all mounts of the same device manually.", mountPath)
		return volume.ReconstructedVolume{}, err
	}
	if err != nil {
		return volume.ReconstructedVolume{}, err
	}

	for _, path := range paths {
		if strings.Contains(path, plugin.host.GetPluginDir(iscsiPluginName)) {
			globalPDPath = path
			break
		}
	}
	// Couldn't fetch globalPDPath
	if len(globalPDPath) == 0 {
		return volume.ReconstructedVolume{}, fmt.Errorf("couldn't fetch globalPDPath. failed to obtain volume spec")
	}

	// Obtain iscsi disk configurations from globalPDPath
	device, _, err := extractDeviceAndPrefix(globalPDPath)
	if err != nil {
		return volume.ReconstructedVolume{}, err
	}
	bkpPortal, iqn, err := extractPortalAndIqn(device)
	if err != nil {
		return volume.ReconstructedVolume{}, err
	}
	arr := strings.Split(device, "-lun-")
	if len(arr) < 2 {
		return volume.ReconstructedVolume{}, fmt.Errorf("failed to retrieve lun from globalPDPath: %v", globalPDPath)
	}
	lun, err := strconv.Atoi(arr[1])
	if err != nil {
		return volume.ReconstructedVolume{}, err
	}
	iface, _ := extractIface(globalPDPath)
	iscsiVolume := &v1.Volume{
		Name: volumeName,
		VolumeSource: v1.VolumeSource{
			ISCSI: &v1.ISCSIVolumeSource{
				TargetPortal:   bkpPortal,
				IQN:            iqn,
				Lun:            int32(lun),
				ISCSIInterface: iface,
			},
		},
	}

	var mountContext string
	if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) {
		kvh, ok := plugin.host.(volume.KubeletVolumeHost)
		if !ok {
			return volume.ReconstructedVolume{}, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
		}
		hu := kvh.GetHostUtil()
		mountContext, err = hu.GetSELinuxMountContext(mountPath)
		if err != nil {
			return volume.ReconstructedVolume{}, err
		}
	}

	return volume.ReconstructedVolume{
		Spec:                volume.NewSpecFromVolume(iscsiVolume),
		SELinuxMountContext: mountContext,
	}, nil
}


ConstructBlockVolumeSpec

func (plugin *iscsiPlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mapPath string) (*volume.Spec, error) {
	pluginDir := plugin.host.GetVolumeDevicePluginDir(iscsiPluginName)
	blkutil := volumepathhandler.NewBlockVolumePathHandler()
	globalMapPathUUID, err := blkutil.FindGlobalMapPathUUIDFromPod(pluginDir, mapPath, podUID)
	if err != nil {
		return nil, err
	}
	klog.V(5).Infof("globalMapPathUUID: %v, err: %v", globalMapPathUUID, err)
	// Retrieve volume information from globalMapPathUUID
	// globalMapPathUUID example:
	// plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{pod uuid}
	// plugins/kubernetes.io/iscsi/volumeDevices/iface-default/192.168.0.10:3260-iqn.2017-05.com.example:test-lun-0/{pod uuid}
	globalMapPath := filepath.Dir(globalMapPathUUID)
	return getVolumeSpecFromGlobalMapPath(volumeName, globalMapPath)
}


GetPath

func (iscsi *iscsiDisk) GetPath() string {
	name := iscsiPluginName
	// safe to use PodVolumeDir now: volume teardown occurs before pod is cleaned up
	return iscsi.plugin.host.GetPodVolumeDir(iscsi.podUID, utilstrings.EscapeQualifiedName(name), iscsi.VolName)
}


iscsiGlobalMapPath

func (iscsi *iscsiDisk) iscsiGlobalMapPath(spec *volume.Spec) (string, error) {
	mounter, err := volumeSpecToMounter(spec, iscsi.plugin.host, iscsi.plugin.targetLocks, nil /* pod */)
	if err != nil {
		klog.Warningf("failed to get iscsi mounter: %v", err)
		return "", err
	}
	return iscsi.manager.MakeGlobalVDPDName(*mounter.iscsiDisk), nil
}


iscsiPodDeviceMapPath

func (iscsi *iscsiDisk) iscsiPodDeviceMapPath() (string, string) {
	name := iscsiPluginName
	return iscsi.plugin.host.GetPodVolumeDeviceDir(iscsi.podUID, utilstrings.EscapeQualifiedName(name)), iscsi.VolName
}


GetAttributes

func (b *iscsiDiskMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:       b.readOnly,
		Managed:        !b.readOnly,
		SELinuxRelabel: !b.mountedWithSELinuxContext,
	}
}


SetUp

func (b *iscsiDiskMounter) SetUp(mounterArgs volume.MounterArgs) error {
	return b.SetUpAt(b.GetPath(), mounterArgs)
}


SetUpAt

func (b *iscsiDiskMounter) SetUpAt(dir string, mounterArgs volume.MounterArgs) error {
	// diskSetUp checks mountpoints and prevent repeated calls
	err := diskSetUp(b.manager, *b, dir, b.mounter, mounterArgs.FsGroup, mounterArgs.FSGroupChangePolicy)
	if err != nil {
		klog.Errorf("iscsi: failed to setup")
	}

	if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) {
		// The volume must have been mounted in MountDevice with -o context.
		// TODO: extract from mount table in GetAttributes() to be sure?
		b.mountedWithSELinuxContext = mounterArgs.SELinuxLabel != ""
	}
	return err
}


TearDown

// Unmounts the bind mount, and detaches the disk only if the disk
// resource was the last reference to that disk on the kubelet.
func (c *iscsiDiskUnmounter) TearDown() error {
	return c.TearDownAt(c.GetPath())
}


TearDownAt

func (c *iscsiDiskUnmounter) TearDownAt(dir string) error {
	return mount.CleanupMountPoint(dir, c.mounter, false)
}


SupportsMetrics

// SupportsMetrics returns true for SupportsMetrics as it initializes the
// MetricsProvider.
func (idm *iscsiDiskMapper) SupportsMetrics() bool {
	return true
}


TearDownDevice

// Even though iSCSI plugin has attacher/detacher implementation, iSCSI plugin
// needs volume detach operation during TearDownDevice(). This method is only
// chance that operations are done on kubelet node during volume teardown sequences.
func (c *iscsiDiskUnmapper) TearDownDevice(mapPath, _ string) error {
	err := c.manager.DetachBlockISCSIDisk(*c, mapPath)
	if err != nil {
		return fmt.Errorf("iscsi: failed to detach disk: %s\nError: %v", mapPath, err)
	}
	klog.V(4).Infof("iscsi: %q is unmounted, deleting the directory", mapPath)
	err = os.RemoveAll(mapPath)
	if err != nil {
		return fmt.Errorf("iscsi: failed to delete the directory: %s\nError: %v", mapPath, err)
	}
	klog.V(4).Infof("iscsi: successfully detached disk: %s", mapPath)
	return nil
}


UnmapPodDevice

func (c *iscsiDiskUnmapper) UnmapPodDevice() error {
	return nil
}


GetGlobalMapPath

// GetGlobalMapPath returns global map path and error
// path: plugins/kubernetes.io/{PluginName}/volumeDevices/{ifaceName}/{portal-some_iqn-lun-lun_id}
func (iscsi *iscsiDisk) GetGlobalMapPath(spec *volume.Spec) (string, error) {
	return iscsi.iscsiGlobalMapPath(spec)
}


GetPodDeviceMapPath

// GetPodDeviceMapPath returns pod device map path and volume name
// path: pods/{podUid}/volumeDevices/kubernetes.io~iscsi
// volumeName: pv0001
func (iscsi *iscsiDisk) GetPodDeviceMapPath() (string, string) {
	return iscsi.iscsiPodDeviceMapPath()
}


portalMounter

func portalMounter(portal string) string {
	if !strings.Contains(portal, ":") {
		portal = portal + ":3260"
	}
	return portal
}


getISCSIVolumeInfo

// get iSCSI volume info: readOnly and fstype
func getISCSIVolumeInfo(spec *volume.Spec) (bool, string, error) {
	// for volume source, readonly is in volume spec
	// for PV, readonly is in PV spec. PV gets the ReadOnly flag indirectly through the PVC source
	if spec.Volume != nil && spec.Volume.ISCSI != nil {
		return spec.Volume.ISCSI.ReadOnly, spec.Volume.ISCSI.FSType, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.ISCSI != nil {
		return spec.ReadOnly, spec.PersistentVolume.Spec.ISCSI.FSType, nil
	}

	return false, "", fmt.Errorf("Spec does not reference an ISCSI volume type")
}


getISCSITargetInfo

// get iSCSI target info: target portal, portals, iqn, and lun
func getISCSITargetInfo(spec *volume.Spec) (string, []string, string, int32, error) {
	if spec.Volume != nil && spec.Volume.ISCSI != nil {
		return spec.Volume.ISCSI.TargetPortal, spec.Volume.ISCSI.Portals, spec.Volume.ISCSI.IQN, spec.Volume.ISCSI.Lun, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.ISCSI != nil {
		return spec.PersistentVolume.Spec.ISCSI.TargetPortal, spec.PersistentVolume.Spec.ISCSI.Portals, spec.PersistentVolume.Spec.ISCSI.IQN, spec.PersistentVolume.Spec.ISCSI.Lun, nil
	}

	return "", nil, "", 0, fmt.Errorf("Spec does not reference an ISCSI volume type")
}


getISCSIInitiatorInfo

// get iSCSI initiator info: iface and initiator name
func getISCSIInitiatorInfo(spec *volume.Spec) (string, *string, error) {
	if spec.Volume != nil && spec.Volume.ISCSI != nil {
		return spec.Volume.ISCSI.ISCSIInterface, spec.Volume.ISCSI.InitiatorName, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.ISCSI != nil {
		return spec.PersistentVolume.Spec.ISCSI.ISCSIInterface, spec.PersistentVolume.Spec.ISCSI.InitiatorName, nil
	}

	return "", nil, fmt.Errorf("Spec does not reference an ISCSI volume type")
}


getISCSIDiscoveryCHAPInfo

// get iSCSI Discovery CHAP boolean
func getISCSIDiscoveryCHAPInfo(spec *volume.Spec) (bool, error) {
	if spec.Volume != nil && spec.Volume.ISCSI != nil {
		return spec.Volume.ISCSI.DiscoveryCHAPAuth, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.ISCSI != nil {
		return spec.PersistentVolume.Spec.ISCSI.DiscoveryCHAPAuth, nil
	}

	return false, fmt.Errorf("Spec does not reference an ISCSI volume type")
}


getISCSISessionCHAPInfo

// get iSCSI Session CHAP boolean
func getISCSISessionCHAPInfo(spec *volume.Spec) (bool, error) {
	if spec.Volume != nil && spec.Volume.ISCSI != nil {
		return spec.Volume.ISCSI.SessionCHAPAuth, nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.ISCSI != nil {
		return spec.PersistentVolume.Spec.ISCSI.SessionCHAPAuth, nil
	}

	return false, fmt.Errorf("Spec does not reference an ISCSI volume type")
}


getISCSISecretNameAndNamespace

// get iSCSI CHAP Secret info: secret name and namespace
func getISCSISecretNameAndNamespace(spec *volume.Spec, defaultSecretNamespace string) (string, string, error) {
	if spec.Volume != nil && spec.Volume.ISCSI != nil {
		if spec.Volume.ISCSI.SecretRef != nil {
			return spec.Volume.ISCSI.SecretRef.Name, defaultSecretNamespace, nil
		}
		return "", "", nil
	} else if spec.PersistentVolume != nil &&
		spec.PersistentVolume.Spec.ISCSI != nil {
		secretRef := spec.PersistentVolume.Spec.ISCSI.SecretRef
		secretNs := defaultSecretNamespace
		if secretRef != nil {
			if len(secretRef.Namespace) != 0 {
				secretNs = secretRef.Namespace
			}
			return secretRef.Name, secretNs, nil
		}
		return "", "", nil
	}

	return "", "", fmt.Errorf("Spec does not reference an ISCSI volume type")
}


createISCSIDisk

func createISCSIDisk(spec *volume.Spec, podUID types.UID, plugin *iscsiPlugin, manager diskManager, secret map[string]string) (*iscsiDisk, error) {
	tp, portals, iqn, lunStr, err := getISCSITargetInfo(spec)
	if err != nil {
		return nil, err
	}

	lun := strconv.Itoa(int(lunStr))
	portal := portalMounter(tp)
	var bkportal []string
	bkportal = append(bkportal, portal)
	for _, p := range portals {
		bkportal = append(bkportal, portalMounter(string(p)))
	}

	iface, initiatorNamePtr, err := getISCSIInitiatorInfo(spec)
	if err != nil {
		return nil, err
	}

	var initiatorName string
	if initiatorNamePtr != nil {
		initiatorName = *initiatorNamePtr
	}
	chapDiscovery, err := getISCSIDiscoveryCHAPInfo(spec)
	if err != nil {
		return nil, err
	}
	chapSession, err := getISCSISessionCHAPInfo(spec)
	if err != nil {
		return nil, err
	}

	initIface := iface
	if initiatorName != "" {
		iface = bkportal[0] + ":" + spec.Name()
	}

	return &iscsiDisk{
		podUID:        podUID,
		VolName:       spec.Name(),
		Portals:       bkportal,
		Iqn:           iqn,
		Lun:           lun,
		InitIface:     initIface,
		Iface:         iface,
		chapDiscovery: chapDiscovery,
		chapSession:   chapSession,
		secret:        secret,
		InitiatorName: initiatorName,
		manager:       manager,
		plugin:        plugin}, nil
}


createSecretMap

func createSecretMap(spec *volume.Spec, plugin *iscsiPlugin, namespace string) (map[string]string, error) {
	var secret map[string]string
	chapDiscover, err := getISCSIDiscoveryCHAPInfo(spec)
	if err != nil {
		return nil, err
	}
	chapSession, err := getISCSISessionCHAPInfo(spec)
	if err != nil {
		return nil, err
	}
	if chapDiscover || chapSession {
		secretName, secretNamespace, err := getISCSISecretNameAndNamespace(spec, namespace)
		if err != nil {
			return nil, err
		}

		if len(secretName) > 0 && len(secretNamespace) > 0 {
			// if secret is provideded, retrieve it
			kubeClient := plugin.host.GetKubeClient()
			if kubeClient == nil {
				return nil, fmt.Errorf("cannot get kube client")
			}
			secretObj, err := kubeClient.CoreV1().Secrets(secretNamespace).Get(context.TODO(), secretName, metav1.GetOptions{})
			if err != nil {
				err = fmt.Errorf("couldn't get secret %v/%v error: %w", secretNamespace, secretName, err)
				return nil, err
			}
			secret = make(map[string]string)
			for name, data := range secretObj.Data {
				klog.V(4).Infof("retrieving CHAP secret name: %s", name)
				secret[name] = string(data)
			}
		}
	}
	return secret, err
}


createPersistentVolumeFromISCSIPVSource

func createPersistentVolumeFromISCSIPVSource(volumeName string, iscsi v1.ISCSIPersistentVolumeSource) *v1.PersistentVolume {
	block := v1.PersistentVolumeBlock
	return &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: volumeName,
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeSource: v1.PersistentVolumeSource{
				ISCSI: &iscsi,
			},
			VolumeMode: &block,
		},
	}
}


getVolumeSpecFromGlobalMapPath

func getVolumeSpecFromGlobalMapPath(volumeName, globalMapPath string) (*volume.Spec, error) {
	// Retrieve volume spec information from globalMapPath
	// globalMapPath example:
	// plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}
	// plugins/kubernetes.io/iscsi/volumeDevices/iface-default/192.168.0.10:3260-iqn.2017-05.com.example:test-lun-0

	// device: 192.168.0.10:3260-iqn.2017-05.com.example:test-lun-0
	device, _, err := extractDeviceAndPrefix(globalMapPath)
	if err != nil {
		return nil, err
	}
	bkpPortal, iqn, err := extractPortalAndIqn(device)
	if err != nil {
		return nil, err
	}
	arr := strings.Split(device, "-lun-")
	if len(arr) < 2 {
		return nil, fmt.Errorf("failed to retrieve lun from globalMapPath: %v", globalMapPath)
	}
	lun, err := strconv.Atoi(arr[1])
	if err != nil {
		return nil, err
	}
	iface, found := extractIface(globalMapPath)
	if !found {
		return nil, fmt.Errorf("failed to retrieve iface from globalMapPath: %v", globalMapPath)
	}
	iscsiPV := createPersistentVolumeFromISCSIPVSource(volumeName,
		v1.ISCSIPersistentVolumeSource{
			TargetPortal:   bkpPortal,
			IQN:            iqn,
			Lun:            int32(lun),
			ISCSIInterface: iface,
		},
	)
	klog.V(5).Infof("ConstructBlockVolumeSpec: TargetPortal: %v, IQN: %v, Lun: %v, ISCSIInterface: %v",
		iscsiPV.Spec.PersistentVolumeSource.ISCSI.TargetPortal,
		iscsiPV.Spec.PersistentVolumeSource.ISCSI.IQN,
		iscsiPV.Spec.PersistentVolumeSource.ISCSI.Lun,
		iscsiPV.Spec.PersistentVolumeSource.ISCSI.ISCSIInterface,
	)
	return volume.NewSpecFromPersistentVolume(iscsiPV, false), nil
}


updateISCSIDiscoverydb

func updateISCSIDiscoverydb(b iscsiDiskMounter, tp string) error {
	if !b.chapDiscovery {
		return nil
	}
	out, err := execWithLog(b, "iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "update", "-n", "discovery.sendtargets.auth.authmethod", "-v", "CHAP")
	if err != nil {
		return fmt.Errorf("iscsi: failed to update discoverydb with CHAP, output: %v", out)
	}

	for _, k := range chapSt {
		v := b.secret[k]
		if len(v) > 0 {
			// explicitly not using execWithLog so secrets are not logged
			out, err := b.exec.Command("iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "update", "-n", k, "-v", v).CombinedOutput()
			if err != nil {
				return fmt.Errorf("iscsi: failed to update discoverydb key %q error: %v", k, string(out))
			}
		}
	}
	return nil
}


updateISCSINode

func updateISCSINode(b iscsiDiskMounter, tp string) error {
	// setting node.session.scan to manual to handle https://github.com/kubernetes/kubernetes/issues/90982
	out, err := execWithLog(b, "iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "-o", "update", "-n", "node.session.scan", "-v", "manual")
	if err != nil {
		// don't fail if iscsiadm fails or the version does not support node.session.scan - log a warning to highlight the potential exposure
		klog.Warningf("iscsi: failed to update node with node.session.scan=manual, possible exposure to issue 90982: %v", out)
	}

	if !b.chapSession {
		return nil
	}

	out, err = execWithLog(b, "iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "-o", "update", "-n", "node.session.auth.authmethod", "-v", "CHAP")
	if err != nil {
		return fmt.Errorf("iscsi: failed to update node with CHAP, output: %v", out)
	}

	for _, k := range chapSess {
		v := b.secret[k]
		if len(v) > 0 {
			// explicitly not using execWithLog so secrets are not logged
			out, err := b.exec.Command("iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "-o", "update", "-n", k, "-v", v).CombinedOutput()
			if err != nil {
				return fmt.Errorf("iscsi: failed to update node session key %q error: %v", k, string(out))
			}
		}
	}
	return nil
}


waitForPathToExist

func waitForPathToExist(devicePath *string, maxRetries int, deviceTransport string) bool {
	// This makes unit testing a lot easier
	return waitForPathToExistInternal(devicePath, maxRetries, deviceTransport, os.Stat, filepath.Glob)
}


waitForPathToExistInternal

func waitForPathToExistInternal(devicePath *string, maxRetries int, deviceTransport string, osStat StatFunc, filepathGlob GlobFunc) bool {
	if devicePath == nil {
		return false
	}

	for i := 0; i < maxRetries; i++ {
		var err error
		if deviceTransport == "tcp" {
			_, err = osStat(*devicePath)
		} else {
			fpath, _ := filepathGlob(*devicePath)
			if fpath == nil {
				err = os.ErrNotExist
			} else {
				// There might be a case that fpath contains multiple device paths if
				// multiple PCI devices connect to same iscsi target. We handle this
				// case at subsequent logic. Pick up only first path here.
				*devicePath = fpath[0]
			}
		}
		if err == nil {
			return true
		}
		if !os.IsNotExist(err) {
			return false
		}
		if i == maxRetries-1 {
			break
		}
		time.Sleep(time.Second)
	}
	return false
}


makePDNameInternal

// make a directory like /var/lib/kubelet/plugins/kubernetes.io/iscsi/iface_name/portal-some_iqn-lun-lun_id
func makePDNameInternal(host volume.VolumeHost, portal string, iqn string, lun string, iface string) string {
	return filepath.Join(host.GetPluginDir(iscsiPluginName), "iface-"+iface, portal+"-"+iqn+"-lun-"+lun)
}


makeVDPDNameInternal

// make a directory like /var/lib/kubelet/plugins/kubernetes.io/iscsi/volumeDevices/iface_name/portal-some_iqn-lun-lun_id
func makeVDPDNameInternal(host volume.VolumeHost, portal string, iqn string, lun string, iface string) string {
	return filepath.Join(host.GetVolumeDevicePluginDir(iscsiPluginName), "iface-"+iface, portal+"-"+iqn+"-lun-"+lun)
}


MakeGlobalPDName

// MakeGlobalPDName returns path of global plugin dir
func (util *ISCSIUtil) MakeGlobalPDName(iscsi iscsiDisk) string {
	return makePDNameInternal(iscsi.plugin.host, iscsi.Portals[0], iscsi.Iqn, iscsi.Lun, iscsi.Iface)
}


MakeGlobalVDPDName

// MakeGlobalVDPDName returns path of global volume device plugin dir
func (util *ISCSIUtil) MakeGlobalVDPDName(iscsi iscsiDisk) string {
	return makeVDPDNameInternal(iscsi.plugin.host, iscsi.Portals[0], iscsi.Iqn, iscsi.Lun, iscsi.Iface)
}


persistISCSIFile

// persistISCSIFile saves iSCSI volume configuration for DetachDisk
// into given directory.
func (util *ISCSIUtil) persistISCSIFile(conf iscsiDisk, mnt string) error {
	file := filepath.Join(mnt, "iscsi.json")
	fp, err := os.Create(file)
	if err != nil {
		return fmt.Errorf("iscsi: create %s err %s", file, err)
	}
	defer fp.Close()
	encoder := json.NewEncoder(fp)
	if err = encoder.Encode(conf); err != nil {
		return fmt.Errorf("iscsi: encode err: %v", err)
	}
	return nil
}


loadISCSI

func (util *ISCSIUtil) loadISCSI(conf *iscsiDisk, mnt string) error {
	file := filepath.Join(mnt, "iscsi.json")
	fp, err := os.Open(file)
	if err != nil {
		return fmt.Errorf("iscsi: open %s err %s", file, err)
	}
	defer fp.Close()
	decoder := json.NewDecoder(fp)
	if err = decoder.Decode(conf); err != nil {
		return fmt.Errorf("iscsi: decode err: %v", err)
	}
	return nil
}


scanOneLun

// scanOneLun scans a single LUN on one SCSI bus
// Use this to avoid scanning the whole SCSI bus for all of the LUNs, which
// would result in the kernel on this node discovering LUNs that it shouldn't
// know about. Extraneous LUNs cause problems because they may get deleted
// without us getting notified, since we were never supposed to know about
// them. When LUNs are deleted without proper cleanup in the kernel, I/O errors
// and timeouts result, which can noticeably degrade performance of future
// operations.
func scanOneLun(hostNumber int, lunNumber int) error {
	filename := fmt.Sprintf("/sys/class/scsi_host/host%d/scan", hostNumber)
	fd, err := os.OpenFile(filename, os.O_WRONLY, 0)
	if err != nil {
		return err
	}
	defer fd.Close()

	// Channel/Target are always 0 for iSCSI
	scanCmd := fmt.Sprintf("0 0 %d", lunNumber)
	if written, err := fd.WriteString(scanCmd); err != nil {
		return err
	} else if 0 == written {
		return fmt.Errorf("no data written to file: %s", filename)
	}

	klog.V(3).Infof("Scanned SCSI host %d LUN %d", hostNumber, lunNumber)
	return nil
}


waitForMultiPathToExist

func waitForMultiPathToExist(devicePaths []string, maxRetries int, deviceUtil volumeutil.DeviceUtil) string {
	if 0 == len(devicePaths) {
		return ""
	}

	for i := 0; i < maxRetries; i++ {
		for _, path := range devicePaths {
			// There shouldn't be any empty device paths. However adding this check
			// for safer side to avoid the possibility of an empty entry.
			if path == "" {
				continue
			}
			// check if the dev is using mpio and if so mount it via the dm-XX device
			if mappedDevicePath := deviceUtil.FindMultipathDeviceForDevice(path); mappedDevicePath != "" {
				return mappedDevicePath
			}
		}
		if i == maxRetries-1 {
			break
		}
		time.Sleep(time.Second)
	}
	return ""
}


AttachDisk

// AttachDisk returns devicePath of volume if attach succeeded otherwise returns error
func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) (string, error) {
	var devicePath string
	devicePaths := map[string]string{}
	var iscsiTransport string
	var lastErr error

	out, err := execWithLog(b, "iscsiadm", "-m", "iface", "-I", b.InitIface, "-o", "show")
	if err != nil {
		klog.Errorf("iscsi: could not read iface %s error: %s", b.InitIface, out)
		return "", err
	}

	iscsiTransport = extractTransportname(out)

	bkpPortal := b.Portals

	// If the initiator name was set, the iface isn't created yet,
	// so create it and copy parameters from the pre-configured one
	if b.InitiatorName != "" {
		if err = cloneIface(b); err != nil {
			klog.Errorf("iscsi: failed to clone iface: %s error: %v", b.InitIface, err)
			return "", err
		}
	}

	// Lock the target while we login to avoid races between 2 volumes that share the same
	// target both logging in or one logging out while another logs in.
	b.plugin.targetLocks.LockKey(b.Iqn)
	defer b.plugin.targetLocks.UnlockKey(b.Iqn)

	// Build a map of SCSI hosts for each target portal. We will need this to
	// issue the bus rescans.
	portalHostMap, err := b.deviceUtil.GetISCSIPortalHostMapForTarget(b.Iqn)
	if err != nil {
		return "", err
	}
	klog.V(4).Infof("AttachDisk portal->host map for %s is %v", b.Iqn, portalHostMap)

	for i := 1; i <= maxAttachAttempts; i++ {
		for _, tp := range bkpPortal {
			if _, found := devicePaths[tp]; found {
				klog.V(4).Infof("Device for portal %q already known", tp)
				continue
			}

			hostNumber, loggedIn := portalHostMap[tp]
			if !loggedIn {
				klog.V(4).Infof("Could not get SCSI host number for portal %s, will attempt login", tp)

				// build discoverydb and discover iscsi target
				execWithLog(b, "iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "new")

				// update discoverydb with CHAP secret
				err = updateISCSIDiscoverydb(b, tp)
				if err != nil {
					lastErr = fmt.Errorf("iscsi: failed to update discoverydb to portal %s error: %v", tp, err)
					continue
				}

				out, err = execWithLog(b, "iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "--discover")
				if err != nil {
					// delete discoverydb record
					execWithLog(b, "iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "delete")
					lastErr = fmt.Errorf("iscsi: failed to sendtargets to portal %s output: %s, err %v", tp, out, err)
					continue
				}

				err = updateISCSINode(b, tp)
				if err != nil {
					// failure to update node db is rare. But deleting record will likely impact those who already start using it.
					lastErr = fmt.Errorf("iscsi: failed to update iscsi node to portal %s error: %v", tp, err)
					continue
				}

				// login to iscsi target
				out, err = execWithLog(b, "iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "--login")
				if err != nil {
					// delete the node record from database
					execWithLog(b, "iscsiadm", "-m", "node", "-p", tp, "-I", b.Iface, "-T", b.Iqn, "-o", "delete")
					lastErr = fmt.Errorf("iscsi: failed to attach disk: Error: %s (%v)", out, err)
					continue
				}

				// in case of node failure/restart, explicitly set to manual login so it doesn't hang on boot
				_, err = execWithLog(b, "iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-o", "update", "-n", "node.startup", "-v", "manual")
				if err != nil {
					// don't fail if we can't set startup mode, but log warning so there is a clue
					klog.Warningf("Warning: Failed to set iSCSI login mode to manual. Error: %v", err)
				}

				// Rebuild the host map after logging in
				portalHostMap, err := b.deviceUtil.GetISCSIPortalHostMapForTarget(b.Iqn)
				if err != nil {
					return "", err
				}
				klog.V(6).Infof("AttachDisk portal->host map for %s is %v", b.Iqn, portalHostMap)

				hostNumber, loggedIn = portalHostMap[tp]
				if !loggedIn {
					klog.Warningf("Could not get SCSI host number for portal %s after logging in", tp)
					continue
				}
			}

			klog.V(5).Infof("AttachDisk: scanning SCSI host %d LUN %s", hostNumber, b.Lun)
			lunNumber, err := strconv.Atoi(b.Lun)
			if err != nil {
				return "", fmt.Errorf("AttachDisk: lun is not a number: %s\nError: %v", b.Lun, err)
			}

			// Scan the iSCSI bus for the LUN
			err = scanOneLun(hostNumber, lunNumber)
			if err != nil {
				return "", err
			}

			if iscsiTransport == "" {
				klog.Errorf("iscsi: could not find transport name in iface %s", b.Iface)
				return "", fmt.Errorf("could not parse iface file for %s", b.Iface)
			}

			addr := tp
			if strings.HasPrefix(tp, "[") {
				// Delete [] from IP address, links in /dev/disk/by-path do not have it.
				addr = strings.NewReplacer("[", "", "]", "").Replace(tp)
			}
			if iscsiTransport == "tcp" {
				devicePath = strings.Join([]string{"/dev/disk/by-path/ip", addr, "iscsi", b.Iqn, "lun", b.Lun}, "-")
			} else {
				devicePath = strings.Join([]string{"/dev/disk/by-path/pci", "*", "ip", addr, "iscsi", b.Iqn, "lun", b.Lun}, "-")
			}

			if exist := waitForPathToExist(&devicePath, deviceDiscoveryTimeout, iscsiTransport); !exist {
				msg := fmt.Sprintf("Timed out waiting for device at path %s after %ds", devicePath, deviceDiscoveryTimeout)
				klog.Error(msg)
				// update last error
				lastErr = errors.New(msg)
				continue
			} else {
				devicePaths[tp] = devicePath
			}
		}
		klog.V(4).Infof("iscsi: tried all devices for %q %d times, %d paths found", b.Iqn, i, len(devicePaths))
		if len(devicePaths) == 0 {
			// No path attached, report error and stop trying. kubelet will try again in a short while
			// delete cloned iface
			execWithLog(b, "iscsiadm", "-m", "iface", "-I", b.Iface, "-o", "delete")
			klog.Errorf("iscsi: failed to get any path for iscsi disk, last err seen:\n%v", lastErr)
			return "", fmt.Errorf("failed to get any path for iscsi disk, last err seen:\n%v", lastErr)
		}
		if len(devicePaths) == len(bkpPortal) {
			// We have all paths
			klog.V(4).Infof("iscsi: all devices for %q found", b.Iqn)
			break
		}
		if len(devicePaths) >= minMultipathCount && i >= minAttachAttempts {
			// We have at least two paths for multipath and we tried the other paths long enough
			klog.V(4).Infof("%d devices found for %q", len(devicePaths), b.Iqn)
			break
		}
	}

	if lastErr != nil {
		klog.Errorf("iscsi: last error occurred during iscsi init:\n%v", lastErr)
	}

	devicePathList := []string{}
	for _, path := range devicePaths {
		devicePathList = append(devicePathList, path)
	}
	// Try to find a multipath device for the volume
	if len(bkpPortal) > 1 {
		// Multipath volume was requested. Wait up to multipathDeviceTimeout seconds for the multipath device to appear.
		devicePath = waitForMultiPathToExist(devicePathList, multipathDeviceTimeout, b.deviceUtil)
	} else {
		// For PVs with 1 portal, just try one time to find the multipath device. This
		// avoids a long pause when the multipath device will never get created, and
		// matches legacy behavior.
		devicePath = waitForMultiPathToExist(devicePathList, 1, b.deviceUtil)
	}

	// When no multipath device is found, just use the first (and presumably only) device
	if devicePath == "" {
		devicePath = devicePathList[0]
	}

	klog.V(5).Infof("iscsi: AttachDisk devicePath: %s", devicePath)

	if err = util.persistISCSI(b); err != nil {
		// Return uncertain error so kubelet calls Unmount / Unmap when the pod
		// is deleted.
		return "", types.NewUncertainProgressError(err.Error())
	}
	return devicePath, nil
}


persistISCSI

// persistISCSI saves iSCSI volume configuration for DetachDisk into global
// mount / map directory.
func (util *ISCSIUtil) persistISCSI(b iscsiDiskMounter) error {
	klog.V(5).Infof("iscsi: AttachDisk volumeMode: %s", b.volumeMode)
	var globalPDPath string
	if b.volumeMode == v1.PersistentVolumeBlock {
		globalPDPath = b.manager.MakeGlobalVDPDName(*b.iscsiDisk)
	} else {
		globalPDPath = b.manager.MakeGlobalPDName(*b.iscsiDisk)
	}

	if err := os.MkdirAll(globalPDPath, 0750); err != nil {
		klog.Errorf("iscsi: failed to mkdir %s, error", globalPDPath)
		return err
	}

	if b.volumeMode == v1.PersistentVolumeFilesystem {
		notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
		if err != nil {
			return err
		}
		if !notMnt {
			// The volume is already mounted, therefore the previous WaitForAttach must have
			// persisted the volume metadata. In addition, the metadata is actually *inside*
			// globalPDPath and we can't write it here, because it was shadowed by the volume
			// mount.
			klog.V(4).Infof("Skipping persistISCSI, the volume is already mounted at %s", globalPDPath)
			return nil
		}
	}

	// Persist iscsi disk config to json file for DetachDisk path
	return util.persistISCSIFile(*(b.iscsiDisk), globalPDPath)
}


deleteDevice

// Delete 1 block device of the form "sd*"
func deleteDevice(deviceName string) error {
	filename := fmt.Sprintf("/sys/block/%s/device/delete", deviceName)
	fd, err := os.OpenFile(filename, os.O_WRONLY, 0)
	if err != nil {
		// The file was not present, so just return without error
		return nil
	}
	defer fd.Close()

	if written, err := fd.WriteString("1"); err != nil {
		return err
	} else if 0 == written {
		return fmt.Errorf("no data written to file: %s", filename)
	}
	klog.V(4).Infof("Deleted block device: %s", deviceName)
	return nil
}


deleteDevices

// deleteDevices tries to remove all the block devices and multipath map devices
// associated with a given iscsi device
func deleteDevices(c iscsiDiskUnmounter) error {
	lunNumber, err := strconv.Atoi(c.iscsiDisk.Lun)
	if err != nil {
		klog.Errorf("iscsi delete devices: lun is not a number: %s\nError: %v", c.iscsiDisk.Lun, err)
		return err
	}
	// Enumerate the devices so we can delete them
	deviceNames, err := c.deviceUtil.FindDevicesForISCSILun(c.iscsiDisk.Iqn, lunNumber)
	if err != nil {
		klog.Errorf("iscsi delete devices: could not get devices associated with LUN %d on target %s\nError: %v",
			lunNumber, c.iscsiDisk.Iqn, err)
		return err
	}
	// Find the multipath device path(s)
	mpathDevices := make(map[string]bool)
	for _, deviceName := range deviceNames {
		path := "/dev/" + deviceName
		// check if the dev is using mpio and if so mount it via the dm-XX device
		if mappedDevicePath := c.deviceUtil.FindMultipathDeviceForDevice(path); mappedDevicePath != "" {
			mpathDevices[mappedDevicePath] = true
		}
	}
	// Flush any multipath device maps
	for mpathDevice := range mpathDevices {
		_, err = c.exec.Command("multipath", "-f", mpathDevice).CombinedOutput()
		if err != nil {
			klog.Warningf("Warning: Failed to flush multipath device map: %s\nError: %v", mpathDevice, err)
			// Fall through -- keep deleting the block devices
		}
		klog.V(4).Infof("Flushed multipath device: %s", mpathDevice)
	}
	for _, deviceName := range deviceNames {
		err = deleteDevice(deviceName)
		if err != nil {
			klog.Warningf("Warning: Failed to delete block device: %s\nError: %v", deviceName, err)
			// Fall through -- keep deleting other block devices
		}
	}
	return nil
}


DetachDisk

// DetachDisk unmounts and detaches a volume from node
func (util *ISCSIUtil) DetachDisk(c iscsiDiskUnmounter, mntPath string) error {
	if pathExists, pathErr := mount.PathExists(mntPath); pathErr != nil {
		return fmt.Errorf("error checking if path exists: %w", pathErr)
	} else if !pathExists {
		klog.Warningf("Warning: Unmount skipped because path does not exist: %v", mntPath)
		return nil
	}

	notMnt, err := c.mounter.IsLikelyNotMountPoint(mntPath)
	if err != nil {
		return err
	}
	if !notMnt {
		if err := c.mounter.Unmount(mntPath); err != nil {
			klog.Errorf("iscsi detach disk: failed to unmount: %s\nError: %v", mntPath, err)
			return err
		}
	}

	// if device is no longer used, see if need to logout the target
	device, _, err := extractDeviceAndPrefix(mntPath)
	if err != nil {
		return err
	}

	var bkpPortal []string
	var volName, iqn, iface, initiatorName string
	found := true

	// load iscsi disk config from json file
	if err := util.loadISCSI(c.iscsiDisk, mntPath); err == nil {
		bkpPortal, iqn, iface, volName = c.iscsiDisk.Portals, c.iscsiDisk.Iqn, c.iscsiDisk.Iface, c.iscsiDisk.VolName
		initiatorName = c.iscsiDisk.InitiatorName
	} else {
		// If the iscsi disk config is not found, fall back to the original behavior.
		// This portal/iqn/iface is no longer referenced, log out.
		// Extract the portal and iqn from device path.
		bkpPortal = make([]string, 1)
		bkpPortal[0], iqn, err = extractPortalAndIqn(device)
		if err != nil {
			return err
		}
		// Extract the iface from the mountPath and use it to log out. If the iface
		// is not found, maintain the previous behavior to facilitate kubelet upgrade.
		// Logout may fail as no session may exist for the portal/IQN on the specified interface.
		iface, found = extractIface(mntPath)
	}

	// Delete all the scsi devices and any multipath devices after unmounting
	if err = deleteDevices(c); err != nil {
		klog.Warningf("iscsi detach disk: failed to delete devices\nError: %v", err)
		// Fall through -- even if deleting fails, a logout may fix problems
	}

	// Lock the target while we determine if we can safely log out or not
	c.plugin.targetLocks.LockKey(iqn)
	defer c.plugin.targetLocks.UnlockKey(iqn)

	portals := removeDuplicate(bkpPortal)
	if len(portals) == 0 {
		return fmt.Errorf("iscsi detach disk: failed to detach iscsi disk. Couldn't get connected portals from configurations")
	}

	// If device is no longer used, see if need to logout the target
	if isSessionBusy(c.iscsiDisk.plugin.host, portals[0], iqn) {
		return nil
	}

	err = util.detachISCSIDisk(c.exec, portals, iqn, iface, volName, initiatorName, found)
	if err != nil {
		return fmt.Errorf("failed to finish detachISCSIDisk, err: %v", err)
	}
	return nil
}


DetachBlockISCSIDisk

// DetachBlockISCSIDisk removes loopback device for a volume and detaches a volume from node
func (util *ISCSIUtil) DetachBlockISCSIDisk(c iscsiDiskUnmapper, mapPath string) error {
	if pathExists, pathErr := mount.PathExists(mapPath); pathErr != nil {
		return fmt.Errorf("error checking if path exists: %w", pathErr)
	} else if !pathExists {
		klog.Warningf("Warning: Unmap skipped because path does not exist: %v", mapPath)
		return nil
	}
	// If we arrive here, device is no longer used, see if need to logout the target
	// device: 192.168.0.10:3260-iqn.2017-05.com.example:test-lun-0
	device, _, err := extractDeviceAndPrefix(mapPath)
	if err != nil {
		return err
	}
	var bkpPortal []string
	var volName, iqn, lun, iface, initiatorName string
	found := true
	// load iscsi disk config from json file
	if err := util.loadISCSI(c.iscsiDisk, mapPath); err == nil {
		bkpPortal, iqn, lun, iface, volName = c.iscsiDisk.Portals, c.iscsiDisk.Iqn, c.iscsiDisk.Lun, c.iscsiDisk.Iface, c.iscsiDisk.VolName
		initiatorName = c.iscsiDisk.InitiatorName
	} else {
		// If the iscsi disk config is not found, fall back to the original behavior.
		// This portal/iqn/iface is no longer referenced, log out.
		// Extract the portal and iqn from device path.
		bkpPortal = make([]string, 1)
		bkpPortal[0], iqn, err = extractPortalAndIqn(device)
		if err != nil {
			return err
		}
		arr := strings.Split(device, "-lun-")
		if len(arr) < 2 {
			return fmt.Errorf("failed to retrieve lun from mapPath: %v", mapPath)
		}
		lun = arr[1]
		// Extract the iface from the mountPath and use it to log out. If the iface
		// is not found, maintain the previous behavior to facilitate kubelet upgrade.
		// Logout may fail as no session may exist for the portal/IQN on the specified interface.
		iface, found = extractIface(mapPath)
	}
	portals := removeDuplicate(bkpPortal)
	if len(portals) == 0 {
		return fmt.Errorf("iscsi detach disk: failed to detach iscsi disk. Couldn't get connected portals from configurations")
	}

	devicePath := getDevByPath(portals[0], iqn, lun)
	klog.V(5).Infof("iscsi: devicePath: %s", devicePath)
	if _, err = os.Stat(devicePath); err != nil {
		return fmt.Errorf("failed to validate devicePath: %s", devicePath)
	}

	// Lock the target while we determine if we can safely log out or not
	c.plugin.targetLocks.LockKey(iqn)
	defer c.plugin.targetLocks.UnlockKey(iqn)

	// If device is no longer used, see if need to logout the target
	if isSessionBusy(c.iscsiDisk.plugin.host, portals[0], iqn) {
		return nil
	}

	// Detach a volume from kubelet node
	err = util.detachISCSIDisk(c.exec, portals, iqn, iface, volName, initiatorName, found)
	if err != nil {
		return fmt.Errorf("failed to finish detachISCSIDisk, err: %v", err)
	}
	return nil
}


detachISCSIDisk

func (util *ISCSIUtil) detachISCSIDisk(exec utilexec.Interface, portals []string, iqn, iface, volName, initiatorName string, found bool) error {
	for _, portal := range portals {
		logoutArgs := []string{"-m", "node", "-p", portal, "-T", iqn, "--logout"}
		deleteArgs := []string{"-m", "node", "-p", portal, "-T", iqn, "-o", "delete"}
		if found {
			logoutArgs = append(logoutArgs, []string{"-I", iface}...)
			deleteArgs = append(deleteArgs, []string{"-I", iface}...)
		}
		klog.Infof("iscsi: log out target %s iqn %s iface %s", portal, iqn, iface)
		out, err := exec.Command("iscsiadm", logoutArgs...).CombinedOutput()
		err = ignoreExitCodes(err, exit_ISCSI_ERR_NO_OBJS_FOUND, exit_ISCSI_ERR_SESS_NOT_FOUND)
		if err != nil {
			klog.Errorf("iscsi: failed to detach disk Error: %s", string(out))
			return err
		}
		// Delete the node record
		klog.Infof("iscsi: delete node record target %s iqn %s", portal, iqn)
		out, err = exec.Command("iscsiadm", deleteArgs...).CombinedOutput()
		err = ignoreExitCodes(err, exit_ISCSI_ERR_NO_OBJS_FOUND, exit_ISCSI_ERR_SESS_NOT_FOUND)
		if err != nil {
			klog.Errorf("iscsi: failed to delete node record Error: %s", string(out))
			return err
		}
	}
	// Delete the iface after all sessions have logged out
	// If the iface is not created via iscsi plugin, skip to delete
	if initiatorName != "" && found && iface == (portals[0]+":"+volName) {
		deleteArgs := []string{"-m", "iface", "-I", iface, "-o", "delete"}
		out, err := exec.Command("iscsiadm", deleteArgs...).CombinedOutput()
		err = ignoreExitCodes(err, exit_ISCSI_ERR_NO_OBJS_FOUND, exit_ISCSI_ERR_SESS_NOT_FOUND)
		if err != nil {
			klog.Errorf("iscsi: failed to delete iface Error: %s", string(out))
			return err
		}
	}

	return nil
}


getDevByPath

func getDevByPath(portal, iqn, lun string) string {
	return "/dev/disk/by-path/ip-" + portal + "-iscsi-" + iqn + "-lun-" + lun
}


extractTransportname

func extractTransportname(ifaceOutput string) (iscsiTransport string) {
	rexOutput := ifaceTransportNameRe.FindStringSubmatch(ifaceOutput)
	if rexOutput == nil {
		return ""
	}
	iscsiTransport = rexOutput[1]

	// While iface.transport_name is a required parameter, handle it being unspecified anyways
	if iscsiTransport == "<empty>" {
		iscsiTransport = "tcp"
	}
	return iscsiTransport
}


extractDeviceAndPrefix

func extractDeviceAndPrefix(mntPath string) (string, string, error) {
	ind := strings.LastIndex(mntPath, "/")
	if ind < 0 {
		return "", "", fmt.Errorf("iscsi detach disk: malformatted mnt path: %s", mntPath)
	}
	device := mntPath[(ind + 1):]
	// strip -lun- from mount path
	ind = strings.LastIndex(mntPath, "-lun-")
	if ind < 0 {
		return "", "", fmt.Errorf("iscsi detach disk: malformatted mnt path: %s", mntPath)
	}
	prefix := mntPath[:ind]
	return device, prefix, nil
}


extractIface

func extractIface(mntPath string) (string, bool) {
	reOutput := ifaceRe.FindStringSubmatch(mntPath)
	if len(reOutput) > 1 {
		return reOutput[1], true
	}

	return "", false
}


extractPortalAndIqn

func extractPortalAndIqn(device string) (string, string, error) {
	ind1 := strings.Index(device, "-")
	if ind1 < 0 {
		return "", "", fmt.Errorf("iscsi detach disk: no portal in %s", device)
	}
	portal := device[0:ind1]
	ind2 := strings.Index(device, "iqn.")
	if ind2 < 0 {
		ind2 = strings.Index(device, "eui.")
	}
	if ind2 < 0 {
		return "", "", fmt.Errorf("iscsi detach disk: no iqn in %s", device)
	}
	ind := strings.LastIndex(device, "-lun-")
	iqn := device[ind2:ind]
	return portal, iqn, nil
}


removeDuplicate

// Remove duplicates or string
func removeDuplicate(s []string) []string {
	m := map[string]bool{}
	for _, v := range s {
		if v != "" && !m[v] {
			s[len(m)] = v
			m[v] = true
		}
	}
	s = s[:len(m)]
	return s
}


parseIscsiadmShow

func parseIscsiadmShow(output string) (map[string]string, error) {
	params := make(map[string]string)
	slice := strings.Split(output, "\n")
	for _, line := range slice {
		if !strings.HasPrefix(line, "iface.") || strings.Contains(line, "<empty>") {
			continue
		}
		iface := strings.Fields(line)
		if len(iface) != 3 || iface[1] != "=" {
			return nil, fmt.Errorf("error: invalid iface setting: %v", iface)
		}
		// iscsi_ifacename is immutable once the iface is created
		if iface[0] == "iface.iscsi_ifacename" {
			continue
		}
		params[iface[0]] = iface[2]
	}
	return params, nil
}


cloneIface

func cloneIface(b iscsiDiskMounter) error {
	var lastErr error
	if b.InitIface == b.Iface {
		return fmt.Errorf("iscsi: cannot clone iface with same name: %s", b.InitIface)
	}
	// get pre-configured iface records
	out, err := execWithLog(b, "iscsiadm", "-m", "iface", "-I", b.InitIface, "-o", "show")
	if err != nil {
		lastErr = fmt.Errorf("iscsi: failed to show iface records: %s (%v)", out, err)
		return lastErr
	}
	// parse obtained records
	params, err := parseIscsiadmShow(out)
	if err != nil {
		lastErr = fmt.Errorf("iscsi: failed to parse iface records: %s (%v)", out, err)
		return lastErr
	}
	// update initiatorname
	params["iface.initiatorname"] = b.InitiatorName
	// create new iface
	out, err = execWithLog(b, "iscsiadm", "-m", "iface", "-I", b.Iface, "-o", "new")
	if err != nil {
		exit, ok := err.(utilexec.ExitError)
		if ok && exit.ExitStatus() == iscsiadmErrorSessExists {
			klog.Infof("iscsi: there is a session already logged in with iface %s", b.Iface)
		} else {
			lastErr = fmt.Errorf("iscsi: failed to create new iface: %s (%v)", out, err)
			return lastErr
		}
	}
	// Get and sort keys to maintain a stable iteration order
	var keys []string
	for k := range params {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	// update new iface records
	for _, key := range keys {
		_, err = execWithLog(b, "iscsiadm", "-m", "iface", "-I", b.Iface, "-o", "update", "-n", key, "-v", params[key])
		if err != nil {
			execWithLog(b, "iscsiadm", "-m", "iface", "-I", b.Iface, "-o", "delete")
			lastErr = fmt.Errorf("iscsi: failed to update iface records: %s (%v). iface(%s) will be used", out, err, b.InitIface)
			break
		}
	}
	return lastErr
}


isSessionBusy

// isSessionBusy determines if the iSCSI session is busy by counting both FS and block volumes in use.
func isSessionBusy(host volume.VolumeHost, portal, iqn string) bool {
	fsDir := host.GetPluginDir(iscsiPluginName)
	countFS, err := getVolCount(fsDir, portal, iqn)
	if err != nil {
		klog.Errorf("iscsi: could not determine FS volumes in use: %v", err)
		return true
	}

	blockDir := host.GetVolumeDevicePluginDir(iscsiPluginName)
	countBlock, err := getVolCount(blockDir, portal, iqn)
	if err != nil {
		klog.Errorf("iscsi: could not determine block volumes in use: %v", err)
		return true
	}

	return countFS+countBlock > 1
}


getVolCount

// getVolCount returns the number of volumes in use by the kubelet.
// It does so by counting the number of directories prefixed by the given portal and IQN.
func getVolCount(dir, portal, iqn string) (int, error) {
	// For FileSystem volumes, the topmost dirs are named after the ifaces, e.g., iface-default or iface-127.0.0.1:3260:pv0.
	// For Block volumes, the default topmost dir is volumeDevices.
	contents, err := ioutil.ReadDir(dir)
	if err != nil {
		if os.IsNotExist(err) {
			return 0, nil
		}
		return 0, err
	}

	// Inside each iface dir, we look for volume dirs prefixed by the given
	// portal + iqn, e.g., 127.0.0.1:3260-iqn.2003-01.io.k8s:e2e.volume-1-lun-2
	var counter int
	for _, c := range contents {
		if !c.IsDir() || c.Name() == config.DefaultKubeletVolumeDevicesDirName {
			continue
		}

		mounts, err := ioutil.ReadDir(filepath.Join(dir, c.Name()))
		if err != nil {
			return 0, err
		}

		for _, m := range mounts {
			volumeMount := m.Name()
			prefix := portal + "-" + iqn
			if strings.HasPrefix(volumeMount, prefix) {
				counter++
			}
		}
	}

	return counter, nil
}


ignoreExitCodes

func ignoreExitCodes(err error, ignoredExitCodes ...int) error {
	exitError, ok := err.(utilexec.ExitError)
	if !ok {
		return err
	}
	for _, code := range ignoredExitCodes {
		if exitError.ExitStatus() == code {
			klog.V(4).Infof("ignored iscsiadm exit code %d", code)
			return nil
		}
	}
	return err
}


execWithLog

func execWithLog(b iscsiDiskMounter, cmd string, args ...string) (string, error) {
	start := time.Now()
	out, err := b.exec.Command(cmd, args...).CombinedOutput()
	if klogV := klog.V(5); klogV.Enabled() {
		d := time.Since(start)
		klogV.Infof("Executed %s %v in %v, err: %v", cmd, args, d, err)
		klogV.Infof("Output: %s", string(out))
	}
	return string(out), err
}


ProbeVolumePlugins

// ProbeVolumePlugins is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&localVolumePlugin{}}
}


Init

func (plugin *localVolumePlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	plugin.volumeLocks = keymutex.NewHashed(0)
	plugin.recorder = host.GetEventRecorder()
	return nil
}


GetPluginName

func (plugin *localVolumePlugin) GetPluginName() string {
	return localVolumePluginName
}


GetVolumeName

func (plugin *localVolumePlugin) GetVolumeName(spec *volume.Spec) (string, error) {
	// This volume is only supported as a PersistentVolumeSource, so the PV name is unique
	return spec.Name(), nil
}


CanSupport

func (plugin *localVolumePlugin) CanSupport(spec *volume.Spec) bool {
	// This volume is only supported as a PersistentVolumeSource
	return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.Local != nil)
}


RequiresRemount

func (plugin *localVolumePlugin) RequiresRemount(spec *volume.Spec) bool {
	return false
}


SupportsMountOption

func (plugin *localVolumePlugin) SupportsMountOption() bool {
	return true
}


SupportsBulkVolumeVerification

func (plugin *localVolumePlugin) SupportsBulkVolumeVerification() bool {
	return false
}


SupportsSELinuxContextMount

func (plugin *localVolumePlugin) SupportsSELinuxContextMount(spec *volume.Spec) (bool, error) {
	return false, nil
}


GetAccessModes

func (plugin *localVolumePlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
	// The current meaning of AccessMode is how many nodes can attach to it, not how many pods can mount it
	return []v1.PersistentVolumeAccessMode{
		v1.ReadWriteOnce,
	}
}


getVolumeSource

func getVolumeSource(spec *volume.Spec) (*v1.LocalVolumeSource, bool, error) {
	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.Local != nil {
		return spec.PersistentVolume.Spec.Local, spec.ReadOnly, nil
	}

	return nil, false, fmt.Errorf("Spec does not reference a Local volume type")
}


NewMounter

func (plugin *localVolumePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
	_, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}

	globalLocalPath, err := plugin.getGlobalLocalPath(spec)
	if err != nil {
		return nil, err
	}

	kvh, ok := plugin.host.(volume.KubeletVolumeHost)
	if !ok {
		return nil, fmt.Errorf("plugin volume host does not implement KubeletVolumeHost interface")
	}

	return &localVolumeMounter{
		localVolume: &localVolume{
			pod:             pod,
			podUID:          pod.UID,
			volName:         spec.Name(),
			mounter:         plugin.host.GetMounter(plugin.GetPluginName()),
			hostUtil:        kvh.GetHostUtil(),
			plugin:          plugin,
			globalPath:      globalLocalPath,
			MetricsProvider: volume.NewMetricsStatFS(plugin.host.GetPodVolumeDir(pod.UID, utilstrings.EscapeQualifiedName(localVolumePluginName), spec.Name())),
		},
		mountOptions: util.MountOptionFromSpec(spec),
		readOnly:     readOnly,
	}, nil

}


NewUnmounter

func (plugin *localVolumePlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
	return &localVolumeUnmounter{
		localVolume: &localVolume{
			podUID:  podUID,
			volName: volName,
			mounter: plugin.host.GetMounter(plugin.GetPluginName()),
			plugin:  plugin,
		},
	}, nil
}


NewBlockVolumeMapper

func (plugin *localVolumePlugin) NewBlockVolumeMapper(spec *volume.Spec, pod *v1.Pod,
	_ volume.VolumeOptions) (volume.BlockVolumeMapper, error) {
	volumeSource, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}

	mapper := &localVolumeMapper{
		localVolume: &localVolume{
			podUID:     pod.UID,
			volName:    spec.Name(),
			globalPath: volumeSource.Path,
			plugin:     plugin,
		},
		readOnly: readOnly,
	}

	blockPath, err := mapper.GetGlobalMapPath(spec)
	if err != nil {
		return nil, fmt.Errorf("failed to get device path: %v", err)
	}
	mapper.MetricsProvider = volume.NewMetricsBlock(filepath.Join(blockPath, string(pod.UID)))

	return mapper, nil
}


NewBlockVolumeUnmapper

func (plugin *localVolumePlugin) NewBlockVolumeUnmapper(volName string,
	podUID types.UID) (volume.BlockVolumeUnmapper, error) {
	return &localVolumeUnmapper{
		localVolume: &localVolume{
			podUID:  podUID,
			volName: volName,
			plugin:  plugin,
		},
	}, nil
}


ConstructVolumeSpec

// TODO: check if no path and no topology constraints are ok
func (plugin *localVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (volume.ReconstructedVolume, error) {
	fs := v1.PersistentVolumeFilesystem
	// The main purpose of reconstructed volume is to clean unused mount points
	// and directories.
	// For filesystem volume with directory source, no global mount path is
	// needed to clean. Empty path is ok.
	// For filesystem volume with block source, we should resolve to its device
	// path if global mount path exists.
	var path string
	mounter := plugin.host.GetMounter(plugin.GetPluginName())
	refs, err := mounter.GetMountRefs(mountPath)
	if err != nil {
		return volume.ReconstructedVolume{}, err
	}
	baseMountPath := plugin.generateBlockDeviceBaseGlobalPath()
	for _, ref := range refs {
		if mount.PathWithinBase(ref, baseMountPath) {
			// If the global mount for block device exists, the source is block
			// device.
			// The resolved device path may not be the exact same as path in
			// local PV object if symbolic link is used. However, it's the true
			// source and can be used in reconstructed volume.
			path, _, err = mount.GetDeviceNameFromMount(mounter, ref)
			if err != nil {
				return volume.ReconstructedVolume{}, err
			}
			klog.V(4).Infof("local: reconstructing volume %q (pod volume mount: %q) with device %q", volumeName, mountPath, path)
			break
		}
	}
	localVolume := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: volumeName,
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeSource: v1.PersistentVolumeSource{
				Local: &v1.LocalVolumeSource{
					Path: path,
				},
			},
			VolumeMode: &fs,
		},
	}
	return volume.ReconstructedVolume{
		Spec: volume.NewSpecFromPersistentVolume(localVolume, false),
	}, nil
}


ConstructBlockVolumeSpec

func (plugin *localVolumePlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName,
	mapPath string) (*volume.Spec, error) {
	block := v1.PersistentVolumeBlock

	localVolume := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: volumeName,
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeSource: v1.PersistentVolumeSource{
				Local: &v1.LocalVolumeSource{
					// Not needed because we don't need to detach local device from the host.
					Path: "",
				},
			},
			VolumeMode: &block,
		},
	}

	return volume.NewSpecFromPersistentVolume(localVolume, false), nil
}


generateBlockDeviceBaseGlobalPath

func (plugin *localVolumePlugin) generateBlockDeviceBaseGlobalPath() string {
	return filepath.Join(plugin.host.GetPluginDir(localVolumePluginName), util.MountsInGlobalPDPath)
}