// Reconstruct volume data structure by reading the pod's volume directories
func (rc *reconciler) reconstructVolume(volume podVolume) (rvolume *reconstructedVolume, rerr error) {
	metrics.ReconstructVolumeOperationsTotal.Inc()
	defer func() {
		if rerr != nil {
			metrics.ReconstructVolumeOperationsErrorsTotal.Inc()
		}
	}()

	// plugin initializations
	plugin, err := rc.volumePluginMgr.FindPluginByName(volume.pluginName)
	if err != nil {
		return nil, err
	}

	// Create pod object
	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			UID: types.UID(volume.podName),
		},
	}
	mapperPlugin, err := rc.volumePluginMgr.FindMapperPluginByName(volume.pluginName)
	if err != nil {
		return nil, err
	}
	if volume.volumeMode == v1.PersistentVolumeBlock && mapperPlugin == nil {
		return nil, fmt.Errorf("could not find block volume plugin %q (spec.Name: %q) pod %q (UID: %q)", volume.pluginName, volume.volumeSpecName, volume.podName, pod.UID)
	}

	reconstructed, err := rc.operationExecutor.ReconstructVolumeOperation(
		volume.volumeMode,
		plugin,
		mapperPlugin,
		pod.UID,
		volume.podName,
		volume.volumeSpecName,
		volume.volumePath,
		volume.pluginName)
	if err != nil {
		return nil, err
	}
	volumeSpec := reconstructed.Spec
	if volumeSpec == nil {
		return nil, fmt.Errorf("failed to reconstruct volume for plugin %q (spec.Name: %q) pod %q (UID: %q): got nil", volume.pluginName, volume.volumeSpecName, volume.podName, pod.UID)
	}

	// We have to find the plugins by volume spec (NOT by plugin name) here
	// in order to correctly reconstruct ephemeral volume types.
	// Searching by spec checks whether the volume is actually attachable
	// (i.e. has a PV) whereas searching by plugin name can only tell whether
	// the plugin supports attachable volumes.
	attachablePlugin, err := rc.volumePluginMgr.FindAttachablePluginBySpec(volumeSpec)
	if err != nil {
		return nil, err
	}
	deviceMountablePlugin, err := rc.volumePluginMgr.FindDeviceMountablePluginBySpec(volumeSpec)
	if err != nil {
		return nil, err
	}

	var uniqueVolumeName v1.UniqueVolumeName
	if attachablePlugin != nil || deviceMountablePlugin != nil {
		uniqueVolumeName, err = util.GetUniqueVolumeNameFromSpec(plugin, volumeSpec)
		if err != nil {
			return nil, err
		}
	} else {
		uniqueVolumeName = util.GetUniqueVolumeNameFromSpecWithPod(volume.podName, plugin, volumeSpec)
	}

	var volumeMapper volumepkg.BlockVolumeMapper
	var volumeMounter volumepkg.Mounter
	var deviceMounter volumepkg.DeviceMounter
	// Path to the mount or block device to check
	var checkPath string

	if volume.volumeMode == v1.PersistentVolumeBlock {
		var newMapperErr error
		volumeMapper, newMapperErr = mapperPlugin.NewBlockVolumeMapper(
			volumeSpec,
			pod,
			volumepkg.VolumeOptions{})
		if newMapperErr != nil {
			return nil, fmt.Errorf(
				"reconstructVolume.NewBlockVolumeMapper failed for volume %q (spec.Name: %q) pod %q (UID: %q) with: %v",
				uniqueVolumeName,
				volumeSpec.Name(),
				volume.podName,
				pod.UID,
				newMapperErr)
		}
		mapDir, linkName := volumeMapper.GetPodDeviceMapPath()
		checkPath = filepath.Join(mapDir, linkName)
	} else {
		var err error
		volumeMounter, err = plugin.NewMounter(
			volumeSpec,
			pod,
			volumepkg.VolumeOptions{})
		if err != nil {
			return nil, fmt.Errorf(
				"reconstructVolume.NewMounter failed for volume %q (spec.Name: %q) pod %q (UID: %q) with: %v",
				uniqueVolumeName,
				volumeSpec.Name(),
				volume.podName,
				pod.UID,
				err)
		}
		checkPath = volumeMounter.GetPath()
		if deviceMountablePlugin != nil {
			deviceMounter, err = deviceMountablePlugin.NewDeviceMounter()
			if err != nil {
				return nil, fmt.Errorf("reconstructVolume.NewDeviceMounter failed for volume %q (spec.Name: %q) pod %q (UID: %q) with: %v",
					uniqueVolumeName,
					volumeSpec.Name(),
					volume.podName,
					pod.UID,
					err)
			}
		}
	}

	// Check existence of mount point for filesystem volume or symbolic link for block volume
	isExist, checkErr := rc.operationExecutor.CheckVolumeExistenceOperation(volumeSpec, checkPath, volumeSpec.Name(), rc.mounter, uniqueVolumeName, volume.podName, pod.UID, attachablePlugin)
	if checkErr != nil {
		return nil, checkErr
	}
	// If mount or symlink doesn't exist, volume reconstruction should be failed
	if !isExist {
		return nil, fmt.Errorf("volume: %q is not mounted", uniqueVolumeName)
	}

	reconstructedVolume := &reconstructedVolume{
		volumeName: uniqueVolumeName,
		podName:    volume.podName,
		volumeSpec: volumeSpec,
		// volume.volumeSpecName is actually InnerVolumeSpecName. It will not be used
		// for volume cleanup.
		// in case pod is added back to desired state, outerVolumeSpecName will be updated from dsw information.
		// See issue #103143 and its fix for details.
		outerVolumeSpecName: volume.volumeSpecName,
		pod:                 pod,
		deviceMounter:       deviceMounter,
		volumeGidValue:      "",
		// devicePath is updated during updateStates() by checking node status's VolumesAttached data.
		// TODO: get device path directly from the volume mount path.
		devicePath:          "",
		mounter:             volumeMounter,
		blockVolumeMapper:   volumeMapper,
		seLinuxMountContext: reconstructed.SELinuxMountContext,
	}
	return reconstructedVolume, nil
}


// readyToUnmount returns true when reconciler can start unmounting volumes.
func (rc *reconciler) readyToUnmount() bool {
	// During kubelet startup, all volumes present on disk are added as uncertain to ASW.
	// Allow unmount only when DSW is fully populated to prevent unmounting volumes that
	// did not reach DSW yet.
	if !rc.populatorHasAddedPods() {
		return false
	}

	// Allow unmount only when ASW device paths were corrected from node.status to prevent
	// calling unmount with a wrong devicePath.
	if len(rc.volumesNeedDevicePath) != 0 {
		return false
	}
	return true
}


// reconstructVolumes tries to reconstruct the actual state of world by scanning all pods' volume
// directories from the disk. For the volumes that cannot support or fail reconstruction, it will
// put the volumes to volumesFailedReconstruction to be cleaned up later when DesiredStateOfWorld
// is populated.
func (rc *reconciler) reconstructVolumes() {
	defer rc.updateLastSyncTime()
	// Get volumes information by reading the pod's directory
	podVolumes, err := getVolumesFromPodDir(rc.kubeletPodsDir)
	if err != nil {
		klog.ErrorS(err, "Cannot get volumes from disk, skip sync states for volume reconstruction")
		return
	}
	reconstructedVolumes := make(map[v1.UniqueVolumeName]*globalVolumeInfo)
	reconstructedVolumeNames := []v1.UniqueVolumeName{}
	for _, volume := range podVolumes {
		if rc.actualStateOfWorld.VolumeExistsWithSpecName(volume.podName, volume.volumeSpecName) {
			klog.V(4).InfoS("Volume exists in actual state, skip cleaning up mounts", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
			// There is nothing to reconstruct
			continue
		}
		reconstructedVolume, err := rc.reconstructVolume(volume)
		if err != nil {
			klog.InfoS("Could not construct volume information", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName, "err", err)
			// We can't reconstruct the volume. Remember to check DSW after it's fully populated and force unmount the volume when it's orphaned.
			rc.volumesFailedReconstruction = append(rc.volumesFailedReconstruction, volume)
			continue
		}
		klog.V(4).InfoS("Adding reconstructed volume to actual state and node status", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
		gvl := &globalVolumeInfo{
			volumeName:        reconstructedVolume.volumeName,
			volumeSpec:        reconstructedVolume.volumeSpec,
			devicePath:        reconstructedVolume.devicePath,
			deviceMounter:     reconstructedVolume.deviceMounter,
			blockVolumeMapper: reconstructedVolume.blockVolumeMapper,
			mounter:           reconstructedVolume.mounter,
		}
		if cachedInfo, ok := reconstructedVolumes[reconstructedVolume.volumeName]; ok {
			gvl = cachedInfo
		}
		gvl.addPodVolume(reconstructedVolume)

		reconstructedVolumeNames = append(reconstructedVolumeNames, reconstructedVolume.volumeName)
		reconstructedVolumes[reconstructedVolume.volumeName] = gvl
	}

	if len(reconstructedVolumes) > 0 {
		// Add the volumes to ASW
		rc.updateStatesNew(reconstructedVolumes)

		// The reconstructed volumes are mounted, hence a previous kubelet must have already put it into node.status.volumesInUse.
		// Remember to update DSW with this information.
		rc.volumesNeedReportedInUse = reconstructedVolumeNames
		// Remember to update devicePath from node.status.volumesAttached
		rc.volumesNeedDevicePath = reconstructedVolumeNames
	}
	klog.V(2).InfoS("Volume reconstruction finished")
}


func (rc *reconciler) updateStatesNew(reconstructedVolumes map[v1.UniqueVolumeName]*globalVolumeInfo) {
	for _, gvl := range reconstructedVolumes {
		err := rc.actualStateOfWorld.MarkVolumeAsAttached(
			//TODO: the devicePath might not be correct for some volume plugins: see issue #54108
			klog.TODO(), gvl.volumeName, gvl.volumeSpec, rc.nodeName, gvl.devicePath)
		if err != nil {
			klog.ErrorS(err, "Could not add volume information to actual state of world", "volumeName", gvl.volumeName)
			continue
		}
		var seLinuxMountContext string
		for _, volume := range gvl.podVolumes {
			markVolumeOpts := operationexecutor.MarkVolumeOpts{
				PodName:             volume.podName,
				PodUID:              types.UID(volume.podName),
				VolumeName:          volume.volumeName,
				Mounter:             volume.mounter,
				BlockVolumeMapper:   volume.blockVolumeMapper,
				OuterVolumeSpecName: volume.outerVolumeSpecName,
				VolumeGidVolume:     volume.volumeGidValue,
				VolumeSpec:          volume.volumeSpec,
				VolumeMountState:    operationexecutor.VolumeMountUncertain,
				SELinuxMountContext: volume.seLinuxMountContext,
			}

			_, err = rc.actualStateOfWorld.CheckAndMarkVolumeAsUncertainViaReconstruction(markVolumeOpts)
			if err != nil {
				klog.ErrorS(err, "Could not add pod to volume information to actual state of world", "pod", klog.KObj(volume.pod))
				continue
			}
			seLinuxMountContext = volume.seLinuxMountContext
			klog.V(2).InfoS("Volume is marked as uncertain and added into the actual state", "pod", klog.KObj(volume.pod), "podName", volume.podName, "volumeName", volume.volumeName, "seLinuxMountContext", volume.seLinuxMountContext)
		}
		// If the volume has device to mount, we mark its device as uncertain.
		if gvl.deviceMounter != nil || gvl.blockVolumeMapper != nil {
			deviceMountPath, err := getDeviceMountPath(gvl)
			if err != nil {
				klog.ErrorS(err, "Could not find device mount path for volume", "volumeName", gvl.volumeName)
				continue
			}
			err = rc.actualStateOfWorld.MarkDeviceAsUncertain(gvl.volumeName, gvl.devicePath, deviceMountPath, seLinuxMountContext)
			if err != nil {
				klog.ErrorS(err, "Could not mark device is uncertain to actual state of world", "volumeName", gvl.volumeName, "deviceMountPath", deviceMountPath)
				continue
			}
			klog.V(2).InfoS("Volume is marked device as uncertain and added into the actual state", "volumeName", gvl.volumeName, "deviceMountPath", deviceMountPath)
		}
	}
}


// cleanOrphanVolumes tries to clean up all volumes that failed reconstruction.
func (rc *reconciler) cleanOrphanVolumes() {
	if len(rc.volumesFailedReconstruction) == 0 {
		return
	}

	for _, volume := range rc.volumesFailedReconstruction {
		if rc.desiredStateOfWorld.VolumeExistsWithSpecName(volume.podName, volume.volumeSpecName) {
			// Some pod needs the volume, don't clean it up and hope that
			// reconcile() calls SetUp and reconstructs the volume in ASW.
			klog.V(4).InfoS("Volume exists in desired state, skip cleaning up mounts", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
			continue
		}
		klog.InfoS("Cleaning up mounts for volume that could not be reconstructed", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
		rc.cleanupMounts(volume)
	}

	klog.V(2).InfoS("Orphan volume cleanup finished")
	// Clean the cache, cleanup is one shot operation.
	rc.volumesFailedReconstruction = make([]podVolume, 0)
}


// updateReconstructedDevicePaths tries to file devicePaths of reconstructed volumes from
// node.Status.VolumesAttached. This can be done only after connection to the API
// server is established, i.e. it can't be part of reconstructVolumes().
func (rc *reconciler) updateReconstructedDevicePaths() {
	klog.V(4).InfoS("Updating reconstructed devicePaths")

	if rc.kubeClient == nil {
		// Skip reconstructing devicePath from node objects if kubelet is in standalone mode.
		// Such kubelet is not expected to mount any attachable volume or Secrets / ConfigMap.
		klog.V(2).InfoS("Skipped reconstruction of DevicePaths from node.status in standalone mode")
		rc.volumesNeedDevicePath = nil
		return
	}

	node, fetchErr := rc.kubeClient.CoreV1().Nodes().Get(context.TODO(), string(rc.nodeName), metav1.GetOptions{})
	if fetchErr != nil {
		// This may repeat few times per second until kubelet is able to read its own status for the first time.
		klog.V(2).ErrorS(fetchErr, "Failed to get Node status to reconstruct device paths")
		return
	}

	for _, volumeID := range rc.volumesNeedDevicePath {
		for _, attachedVolume := range node.Status.VolumesAttached {
			if volumeID != attachedVolume.Name {
				continue
			}
			rc.actualStateOfWorld.UpdateReconstructedDevicePath(volumeID, attachedVolume.DevicePath)
			klog.V(4).InfoS("Updated devicePath from node status for volume", "volumeName", attachedVolume.Name, "path", attachedVolume.DevicePath)
		}
	}
	klog.V(2).InfoS("DevicePaths of reconstructed volumes updated")
	rc.volumesNeedDevicePath = nil
}


func NewVolumeManager(

// NewVolumeManager returns a new concrete instance implementing the
// VolumeManager interface.
//
// kubeClient - kubeClient is the kube API client used by DesiredStateOfWorldPopulator
// to communicate with the API server to fetch PV and PVC objects
//
// volumePluginMgr - the volume plugin manager used to access volume plugins.
// Must be pre-initialized.
func NewVolumeManager(
	controllerAttachDetachEnabled bool,
	nodeName k8stypes.NodeName,
	podManager pod.Manager,
	podStateProvider podStateProvider,
	kubeClient clientset.Interface,
	volumePluginMgr *volume.VolumePluginMgr,
	kubeContainerRuntime container.Runtime,
	mounter mount.Interface,
	hostutil hostutil.HostUtils,
	kubeletPodsDir string,
	recorder record.EventRecorder,
	keepTerminatedPodVolumes bool,
	blockVolumePathHandler volumepathhandler.BlockVolumePathHandler) VolumeManager {

	seLinuxTranslator := util.NewSELinuxLabelTranslator()
	vm := &volumeManager{
		kubeClient:          kubeClient,
		volumePluginMgr:     volumePluginMgr,
		desiredStateOfWorld: cache.NewDesiredStateOfWorld(volumePluginMgr, seLinuxTranslator),
		actualStateOfWorld:  cache.NewActualStateOfWorld(nodeName, volumePluginMgr),
		operationExecutor: operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
			kubeClient,
			volumePluginMgr,
			recorder,
			blockVolumePathHandler)),
	}

	intreeToCSITranslator := csitrans.New()
	csiMigratedPluginManager := csimigration.NewPluginManager(intreeToCSITranslator, utilfeature.DefaultFeatureGate)

	vm.intreeToCSITranslator = intreeToCSITranslator
	vm.csiMigratedPluginManager = csiMigratedPluginManager
	vm.desiredStateOfWorldPopulator = populator.NewDesiredStateOfWorldPopulator(
		kubeClient,
		desiredStateOfWorldPopulatorLoopSleepPeriod,
		podManager,
		podStateProvider,
		vm.desiredStateOfWorld,
		vm.actualStateOfWorld,
		kubeContainerRuntime,
		keepTerminatedPodVolumes,
		csiMigratedPluginManager,
		intreeToCSITranslator,
		volumePluginMgr)
	vm.reconciler = reconciler.NewReconciler(
		kubeClient,
		controllerAttachDetachEnabled,
		reconcilerLoopSleepPeriod,
		waitForAttachTimeout,
		nodeName,
		vm.desiredStateOfWorld,
		vm.actualStateOfWorld,
		vm.desiredStateOfWorldPopulator.HasAddedPods,
		vm.operationExecutor,
		mounter,
		hostutil,
		volumePluginMgr,
		kubeletPodsDir)

	return vm
}


func (vm *volumeManager) Run(sourcesReady config.SourcesReady, stopCh <-chan struct{}) {
	defer runtime.HandleCrash()

	if vm.kubeClient != nil {
		// start informer for CSIDriver
		go vm.volumePluginMgr.Run(stopCh)
	}

	go vm.desiredStateOfWorldPopulator.Run(sourcesReady, stopCh)
	klog.V(2).InfoS("The desired_state_of_world populator starts")

	klog.InfoS("Starting Kubelet Volume Manager")
	go vm.reconciler.Run(stopCh)

	metrics.Register(vm.actualStateOfWorld, vm.desiredStateOfWorld, vm.volumePluginMgr)

	<-stopCh
	klog.InfoS("Shutting down Kubelet Volume Manager")
}


func (vm *volumeManager) GetMountedVolumesForPod(podName types.UniquePodName) container.VolumeMap {
	podVolumes := make(container.VolumeMap)
	for _, mountedVolume := range vm.actualStateOfWorld.GetMountedVolumesForPod(podName) {
		podVolumes[mountedVolume.OuterVolumeSpecName] = container.VolumeInfo{
			Mounter:             mountedVolume.Mounter,
			BlockVolumeMapper:   mountedVolume.BlockVolumeMapper,
			ReadOnly:            mountedVolume.VolumeSpec.ReadOnly,
			InnerVolumeSpecName: mountedVolume.InnerVolumeSpecName,
		}
	}
	return podVolumes
}


func (vm *volumeManager) GetPossiblyMountedVolumesForPod(podName types.UniquePodName) container.VolumeMap {
	podVolumes := make(container.VolumeMap)
	for _, mountedVolume := range vm.actualStateOfWorld.GetPossiblyMountedVolumesForPod(podName) {
		podVolumes[mountedVolume.OuterVolumeSpecName] = container.VolumeInfo{
			Mounter:             mountedVolume.Mounter,
			BlockVolumeMapper:   mountedVolume.BlockVolumeMapper,
			ReadOnly:            mountedVolume.VolumeSpec.ReadOnly,
			InnerVolumeSpecName: mountedVolume.InnerVolumeSpecName,
		}
	}
	return podVolumes
}


func (vm *volumeManager) GetExtraSupplementalGroupsForPod(pod *v1.Pod) []int64 {
	podName := util.GetUniquePodName(pod)
	supplementalGroups := sets.NewString()

	for _, mountedVolume := range vm.actualStateOfWorld.GetMountedVolumesForPod(podName) {
		if mountedVolume.VolumeGidValue != "" {
			supplementalGroups.Insert(mountedVolume.VolumeGidValue)
		}
	}

	result := make([]int64, 0, supplementalGroups.Len())
	for _, group := range supplementalGroups.List() {
		iGroup, extra := getExtraSupplementalGid(group, pod)
		if !extra {
			continue
		}

		result = append(result, int64(iGroup))
	}

	return result
}


func (vm *volumeManager) GetVolumesInUse() []v1.UniqueVolumeName {
	// Report volumes in desired state of world and actual state of world so
	// that volumes are marked in use as soon as the decision is made that the
	// volume *should* be attached to this node until it is safely unmounted.
	desiredVolumes := vm.desiredStateOfWorld.GetVolumesToMount()
	allAttachedVolumes := vm.actualStateOfWorld.GetAttachedVolumes()
	volumesToReportInUse := make([]v1.UniqueVolumeName, 0, len(desiredVolumes)+len(allAttachedVolumes))
	desiredVolumesMap := make(map[v1.UniqueVolumeName]bool, len(desiredVolumes)+len(allAttachedVolumes))

	for _, volume := range desiredVolumes {
		if volume.PluginIsAttachable {
			if _, exists := desiredVolumesMap[volume.VolumeName]; !exists {
				desiredVolumesMap[volume.VolumeName] = true
				volumesToReportInUse = append(volumesToReportInUse, volume.VolumeName)
			}
		}
	}

	for _, volume := range allAttachedVolumes {
		if volume.PluginIsAttachable {
			if _, exists := desiredVolumesMap[volume.VolumeName]; !exists {
				volumesToReportInUse = append(volumesToReportInUse, volume.VolumeName)
			}
		}
	}

	sort.Slice(volumesToReportInUse, func(i, j int) bool {
		return string(volumesToReportInUse[i]) < string(volumesToReportInUse[j])
	})
	return volumesToReportInUse
}


func (vm *volumeManager) ReconcilerStatesHasBeenSynced() bool {
	return vm.reconciler.StatesHasBeenSynced()
}


func (vm *volumeManager) VolumeIsAttached(
	volumeName v1.UniqueVolumeName) bool {
	return vm.actualStateOfWorld.VolumeExists(volumeName)
}


func (vm *volumeManager) MarkVolumesAsReportedInUse(
	volumesReportedAsInUse []v1.UniqueVolumeName) {
	vm.desiredStateOfWorld.MarkVolumesReportedInUse(volumesReportedAsInUse)
}


func (vm *volumeManager) WaitForAttachAndMount(pod *v1.Pod) error {
	if pod == nil {
		return nil
	}

	expectedVolumes := getExpectedVolumes(pod)
	if len(expectedVolumes) == 0 {
		// No volumes to verify
		return nil
	}

	klog.V(3).InfoS("Waiting for volumes to attach and mount for pod", "pod", klog.KObj(pod))
	uniquePodName := util.GetUniquePodName(pod)

	// Some pods expect to have Setup called over and over again to update.
	// Remount plugins for which this is true. (Atomically updating volumes,
	// like Downward API, depend on this to update the contents of the volume).
	vm.desiredStateOfWorldPopulator.ReprocessPod(uniquePodName)

	err := wait.PollImmediate(
		podAttachAndMountRetryInterval,
		podAttachAndMountTimeout,
		vm.verifyVolumesMountedFunc(uniquePodName, expectedVolumes))

	if err != nil {
		unmountedVolumes :=
			vm.getUnmountedVolumes(uniquePodName, expectedVolumes)
		// Also get unattached volumes and volumes not in dsw for error message
		unattachedVolumes :=
			vm.getUnattachedVolumes(uniquePodName)
		volumesNotInDSW :=
			vm.getVolumesNotInDSW(uniquePodName, expectedVolumes)

		if len(unmountedVolumes) == 0 {
			return nil
		}

		return fmt.Errorf(
			"unmounted volumes=%v, unattached volumes=%v, failed to process volumes=%v: %s",
			unmountedVolumes,
			unattachedVolumes,
			volumesNotInDSW,
			err)
	}

	klog.V(3).InfoS("All volumes are attached and mounted for pod", "pod", klog.KObj(pod))
	return nil
}


func (vm *volumeManager) WaitForUnmount(pod *v1.Pod) error {
	if pod == nil {
		return nil
	}

	klog.V(3).InfoS("Waiting for volumes to unmount for pod", "pod", klog.KObj(pod))
	uniquePodName := util.GetUniquePodName(pod)

	vm.desiredStateOfWorldPopulator.ReprocessPod(uniquePodName)

	err := wait.PollImmediate(
		podAttachAndMountRetryInterval,
		podAttachAndMountTimeout,
		vm.verifyVolumesUnmountedFunc(uniquePodName))

	if err != nil {
		var mountedVolumes []string
		for _, v := range vm.actualStateOfWorld.GetMountedVolumesForPod(uniquePodName) {
			mountedVolumes = append(mountedVolumes, v.OuterVolumeSpecName)
		}
		sort.Strings(mountedVolumes)

		if len(mountedVolumes) == 0 {
			return nil
		}

		return fmt.Errorf(
			"mounted volumes=%v: %s",
			mountedVolumes,
			err)
	}

	klog.V(3).InfoS("All volumes are unmounted for pod", "pod", klog.KObj(pod))
	return nil
}


func (vm *volumeManager) getVolumesNotInDSW(uniquePodName types.UniquePodName, expectedVolumes []string) []string {
	volumesNotInDSW := sets.NewString(expectedVolumes...)

	for _, volumeToMount := range vm.desiredStateOfWorld.GetVolumesToMount() {
		if volumeToMount.PodName == uniquePodName {
			volumesNotInDSW.Delete(volumeToMount.OuterVolumeSpecName)
		}
	}

	return volumesNotInDSW.List()
}


// getUnattachedVolumes returns a list of the volumes that are expected to be attached but
// are not currently attached to the node
func (vm *volumeManager) getUnattachedVolumes(uniquePodName types.UniquePodName) []string {
	unattachedVolumes := []string{}
	for _, volumeToMount := range vm.desiredStateOfWorld.GetVolumesToMount() {
		if volumeToMount.PodName == uniquePodName &&
			volumeToMount.PluginIsAttachable &&
			!vm.actualStateOfWorld.VolumeExists(volumeToMount.VolumeName) {
			unattachedVolumes = append(unattachedVolumes, volumeToMount.OuterVolumeSpecName)
		}
	}

	return unattachedVolumes
}


// verifyVolumesMountedFunc returns a method that returns true when all expected
// volumes are mounted.
func (vm *volumeManager) verifyVolumesMountedFunc(podName types.UniquePodName, expectedVolumes []string) wait.ConditionFunc {
	return func() (done bool, err error) {
		if errs := vm.desiredStateOfWorld.PopPodErrors(podName); len(errs) > 0 {
			return true, errors.New(strings.Join(errs, "; "))
		}
		return len(vm.getUnmountedVolumes(podName, expectedVolumes)) == 0, nil
	}
}


// verifyVolumesUnmountedFunc returns a method that is true when there are no mounted volumes for this
// pod.
func (vm *volumeManager) verifyVolumesUnmountedFunc(podName types.UniquePodName) wait.ConditionFunc {
	return func() (done bool, err error) {
		if errs := vm.desiredStateOfWorld.PopPodErrors(podName); len(errs) > 0 {
			return true, errors.New(strings.Join(errs, "; "))
		}
		return len(vm.actualStateOfWorld.GetMountedVolumesForPod(podName)) == 0, nil
	}
}


// getUnmountedVolumes fetches the current list of mounted volumes from
// the actual state of the world, and uses it to process the list of
// expectedVolumes. It returns a list of unmounted volumes.
// The list also includes volume that may be mounted in uncertain state.
func (vm *volumeManager) getUnmountedVolumes(podName types.UniquePodName, expectedVolumes []string) []string {
	mountedVolumes := sets.NewString()
	for _, mountedVolume := range vm.actualStateOfWorld.GetMountedVolumesForPod(podName) {
		mountedVolumes.Insert(mountedVolume.OuterVolumeSpecName)
	}
	return filterUnmountedVolumes(mountedVolumes, expectedVolumes)
}


// filterUnmountedVolumes adds each element of expectedVolumes that is not in
// mountedVolumes to a list of unmountedVolumes and returns it.
func filterUnmountedVolumes(mountedVolumes sets.String, expectedVolumes []string) []string {
	unmountedVolumes := []string{}
	for _, expectedVolume := range expectedVolumes {
		if !mountedVolumes.Has(expectedVolume) {
			unmountedVolumes = append(unmountedVolumes, expectedVolume)
		}
	}
	return unmountedVolumes
}


// getExpectedVolumes returns a list of volumes that must be mounted in order to
// consider the volume setup step for this pod satisfied.
func getExpectedVolumes(pod *v1.Pod) []string {
	mounts, devices, _ := util.GetPodVolumeNames(pod)
	return mounts.Union(devices).UnsortedList()
}


// getExtraSupplementalGid returns the value of an extra supplemental GID as
// defined by an annotation on a volume and a boolean indicating whether the
// volume defined a GID that the pod doesn't already request.
func getExtraSupplementalGid(volumeGidValue string, pod *v1.Pod) (int64, bool) {
	if volumeGidValue == "" {
		return 0, false
	}

	gid, err := strconv.ParseInt(volumeGidValue, 10, 64)
	if err != nil {
		return 0, false
	}

	if pod.Spec.SecurityContext != nil {
		for _, existingGid := range pod.Spec.SecurityContext.SupplementalGroups {
			if gid == int64(existingGid) {
				return 0, false
			}
		}
	}

	return gid, true
}


func newNetworkCounters() (*networkCounter, error) {
	packetsReceivedPerSecondCounter, err := newPerfCounter(packetsReceivedPerSecondQuery)
	if err != nil {
		return nil, err
	}

	packetsSentPerSecondCounter, err := newPerfCounter(packetsSentPerSecondQuery)
	if err != nil {
		return nil, err
	}

	bytesReceivedPerSecondCounter, err := newPerfCounter(bytesReceivedPerSecondQuery)
	if err != nil {
		return nil, err
	}

	bytesSentPerSecondCounter, err := newPerfCounter(bytesSentPerSecondQuery)
	if err != nil {
		return nil, err
	}

	packetsReceivedDiscardedCounter, err := newPerfCounter(packetsReceivedDiscardedQuery)
	if err != nil {
		return nil, err
	}

	packetsReceivedErrorsCounter, err := newPerfCounter(packetsReceivedErrorsQuery)
	if err != nil {
		return nil, err
	}

	packetsOutboundDiscardedCounter, err := newPerfCounter(packetsOutboundDiscardedQuery)
	if err != nil {
		return nil, err
	}

	packetsOutboundErrorsCounter, err := newPerfCounter(packetsOutboundErrorsQuery)
	if err != nil {
		return nil, err
	}

	return &networkCounter{
		packetsReceivedPerSecondCounter: packetsReceivedPerSecondCounter,
		packetsSentPerSecondCounter:     packetsSentPerSecondCounter,
		bytesReceivedPerSecondCounter:   bytesReceivedPerSecondCounter,
		bytesSentPerSecondCounter:       bytesSentPerSecondCounter,
		packetsReceivedDiscardedCounter: packetsReceivedDiscardedCounter,
		packetsReceivedErrorsCounter:    packetsReceivedErrorsCounter,
		packetsOutboundDiscardedCounter: packetsOutboundDiscardedCounter,
		packetsOutboundErrorsCounter:    packetsOutboundErrorsCounter,
		adapterStats:                    map[string]cadvisorapi.InterfaceStats{},
	}, nil
}


func (n *networkCounter) getData() ([]cadvisorapi.InterfaceStats, error) {
	packetsReceivedPerSecondData, err := n.packetsReceivedPerSecondCounter.getDataList()
	if err != nil {
		klog.ErrorS(err, "Unable to get packetsReceivedPerSecond perf counter data")
		return nil, err
	}

	packetsSentPerSecondData, err := n.packetsSentPerSecondCounter.getDataList()
	if err != nil {
		klog.ErrorS(err, "Unable to get packetsSentPerSecond perf counter data")
		return nil, err
	}

	bytesReceivedPerSecondData, err := n.bytesReceivedPerSecondCounter.getDataList()
	if err != nil {
		klog.ErrorS(err, "Unable to get bytesReceivedPerSecond perf counter data")
		return nil, err
	}

	bytesSentPerSecondData, err := n.bytesSentPerSecondCounter.getDataList()
	if err != nil {
		klog.ErrorS(err, "Unable to get bytesSentPerSecond perf counter data")
		return nil, err
	}

	packetsReceivedDiscardedData, err := n.packetsReceivedDiscardedCounter.getDataList()
	if err != nil {
		klog.ErrorS(err, "Unable to get packetsReceivedDiscarded perf counter data")
		return nil, err
	}

	packetsReceivedErrorsData, err := n.packetsReceivedErrorsCounter.getDataList()
	if err != nil {
		klog.ErrorS(err, "Unable to get packetsReceivedErrors perf counter data")
		return nil, err
	}

	packetsOutboundDiscardedData, err := n.packetsOutboundDiscardedCounter.getDataList()
	if err != nil {
		klog.ErrorS(err, "Unable to get packetsOutboundDiscarded perf counter data")
		return nil, err
	}

	packetsOutboundErrorsData, err := n.packetsOutboundErrorsCounter.getDataList()
	if err != nil {
		klog.ErrorS(err, "Unable to get packetsOutboundErrors perf counter data")
		return nil, err
	}

	n.mu.Lock()
	defer n.mu.Unlock()
	n.mergeCollectedData(
		packetsReceivedPerSecondData,
		packetsSentPerSecondData,
		bytesReceivedPerSecondData,
		bytesSentPerSecondData,
		packetsReceivedDiscardedData,
		packetsReceivedErrorsData,
		packetsOutboundDiscardedData,
		packetsOutboundErrorsData,
	)
	return n.listInterfaceStats(), nil
}


// mergeCollectedData merges the collected data into cache. It should be invoked under lock protected.
func (n *networkCounter) mergeCollectedData(packetsReceivedPerSecondData,
	packetsSentPerSecondData,
	bytesReceivedPerSecondData,
	bytesSentPerSecondData,
	packetsReceivedDiscardedData,
	packetsReceivedErrorsData,
	packetsOutboundDiscardedData,
	packetsOutboundErrorsData map[string]uint64) {
	adapters := sets.NewString()

	// merge the collected data and list of adapters.
	adapters.Insert(n.mergePacketsReceivedPerSecondData(packetsReceivedPerSecondData)...)
	adapters.Insert(n.mergePacketsSentPerSecondData(packetsSentPerSecondData)...)
	adapters.Insert(n.mergeBytesReceivedPerSecondData(bytesReceivedPerSecondData)...)
	adapters.Insert(n.mergeBytesSentPerSecondData(bytesSentPerSecondData)...)
	adapters.Insert(n.mergePacketsReceivedDiscardedData(packetsReceivedDiscardedData)...)
	adapters.Insert(n.mergePacketsReceivedErrorsData(packetsReceivedErrorsData)...)
	adapters.Insert(n.mergePacketsOutboundDiscardedData(packetsOutboundDiscardedData)...)
	adapters.Insert(n.mergePacketsOutboundErrorsData(packetsOutboundErrorsData)...)

	// delete the cache for non-existing adapters.
	for adapter := range n.adapterStats {
		if !adapters.Has(adapter) {
			delete(n.adapterStats, adapter)
		}
	}
}


func (n *networkCounter) mergePacketsReceivedPerSecondData(packetsReceivedPerSecondData map[string]uint64) []string {
	var adapters []string
	for adapterName, value := range packetsReceivedPerSecondData {
		adapters = append(adapters, adapterName)
		newStat := n.adapterStats[adapterName]
		newStat.Name = adapterName
		newStat.RxPackets = newStat.RxPackets + value
		n.adapterStats[adapterName] = newStat
	}

	return adapters
}


func (n *networkCounter) mergePacketsSentPerSecondData(packetsSentPerSecondData map[string]uint64) []string {
	var adapters []string
	for adapterName, value := range packetsSentPerSecondData {
		adapters = append(adapters, adapterName)
		newStat := n.adapterStats[adapterName]
		newStat.Name = adapterName
		newStat.TxPackets = newStat.TxPackets + value
		n.adapterStats[adapterName] = newStat
	}

	return adapters
}


func (n *networkCounter) mergeBytesReceivedPerSecondData(bytesReceivedPerSecondData map[string]uint64) []string {
	var adapters []string
	for adapterName, value := range bytesReceivedPerSecondData {
		adapters = append(adapters, adapterName)
		newStat := n.adapterStats[adapterName]
		newStat.Name = adapterName
		newStat.RxBytes = newStat.RxBytes + value
		n.adapterStats[adapterName] = newStat
	}

	return adapters
}


func (n *networkCounter) mergeBytesSentPerSecondData(bytesSentPerSecondData map[string]uint64) []string {
	var adapters []string
	for adapterName, value := range bytesSentPerSecondData {
		adapters = append(adapters, adapterName)
		newStat := n.adapterStats[adapterName]
		newStat.Name = adapterName
		newStat.TxBytes = newStat.TxBytes + value
		n.adapterStats[adapterName] = newStat
	}

	return adapters
}


func (n *networkCounter) mergePacketsReceivedDiscardedData(packetsReceivedDiscardedData map[string]uint64) []string {
	var adapters []string
	for adapterName, value := range packetsReceivedDiscardedData {
		adapters = append(adapters, adapterName)
		newStat := n.adapterStats[adapterName]
		newStat.Name = adapterName
		newStat.RxDropped = value
		n.adapterStats[adapterName] = newStat
	}

	return adapters
}


func (n *networkCounter) mergePacketsReceivedErrorsData(packetsReceivedErrorsData map[string]uint64) []string {
	var adapters []string
	for adapterName, value := range packetsReceivedErrorsData {
		adapters = append(adapters, adapterName)
		newStat := n.adapterStats[adapterName]
		newStat.Name = adapterName
		newStat.RxErrors = value
		n.adapterStats[adapterName] = newStat
	}

	return adapters
}


func (n *networkCounter) mergePacketsOutboundDiscardedData(packetsOutboundDiscardedData map[string]uint64) []string {
	var adapters []string
	for adapterName, value := range packetsOutboundDiscardedData {
		adapters = append(adapters, adapterName)
		newStat := n.adapterStats[adapterName]
		newStat.Name = adapterName
		newStat.TxDropped = value
		n.adapterStats[adapterName] = newStat
	}

	return adapters
}


func (n *networkCounter) mergePacketsOutboundErrorsData(packetsOutboundErrorsData map[string]uint64) []string {
	var adapters []string
	for adapterName, value := range packetsOutboundErrorsData {
		adapters = append(adapters, adapterName)
		newStat := n.adapterStats[adapterName]
		newStat.Name = adapterName
		newStat.TxErrors = value
		n.adapterStats[adapterName] = newStat
	}

	return adapters
}


func (n *networkCounter) listInterfaceStats() []cadvisorapi.InterfaceStats {
	stats := make([]cadvisorapi.InterfaceStats, 0, len(n.adapterStats))
	for _, stat := range n.adapterStats {
		stats = append(stats, stat)
	}
	return stats
}


// NewPerfCounterClient creates a client using perf counters
func NewPerfCounterClient() (Client, error) {
	// Initialize the cache
	initCache := cpuUsageCoreNanoSecondsCache{0, 0}
	return newClient(&perfCounterNodeStatsClient{
		cpuUsageCoreNanoSecondsCache: initCache,
	})
}


func (p *perfCounterNodeStatsClient) startMonitoring() error {
	memory, err := getPhysicallyInstalledSystemMemoryBytes()
	if err != nil {
		return err
	}

	osInfo, err := GetOSInfo()
	if err != nil {
		return err
	}

	p.nodeInfo = nodeInfo{
		kernelVersion:               osInfo.GetPatchVersion(),
		osImageVersion:              osInfo.ProductName,
		memoryPhysicalCapacityBytes: memory,
		startTime:                   time.Now(),
	}

	cpuCounter, err := newPerfCounter(cpuQuery)
	if err != nil {
		return err
	}

	memWorkingSetCounter, err := newPerfCounter(memoryPrivWorkingSetQuery)
	if err != nil {
		return err
	}

	memCommittedBytesCounter, err := newPerfCounter(memoryCommittedBytesQuery)
	if err != nil {
		return err
	}

	networkAdapterCounter, err := newNetworkCounters()
	if err != nil {
		return err
	}

	go wait.Forever(func() {
		p.collectMetricsData(cpuCounter, memWorkingSetCounter, memCommittedBytesCounter, networkAdapterCounter)
	}, perfCounterUpdatePeriod)

	// Cache the CPU usage every defaultCachePeriod
	go wait.Forever(func() {
		newValue := p.nodeMetrics.cpuUsageCoreNanoSeconds
		p.mu.Lock()
		defer p.mu.Unlock()
		p.cpuUsageCoreNanoSecondsCache = cpuUsageCoreNanoSecondsCache{
			previousValue: p.cpuUsageCoreNanoSecondsCache.latestValue,
			latestValue:   newValue,
		}
	}, defaultCachePeriod)

	return nil
}


func (p *perfCounterNodeStatsClient) getMachineInfo() (*cadvisorapi.MachineInfo, error) {
	hostname, err := os.Hostname()
	if err != nil {
		return nil, err
	}

	systemUUID, err := getSystemUUID()
	if err != nil {
		return nil, err
	}

	bootId, err := getBootID()
	if err != nil {
		return nil, err
	}

	return &cadvisorapi.MachineInfo{
		NumCores:       ProcessorCount(),
		MemoryCapacity: p.nodeInfo.memoryPhysicalCapacityBytes,
		MachineID:      hostname,
		SystemUUID:     systemUUID,
		BootID:         bootId,
	}, nil
}


func ProcessorCount() int

// runtime.NumCPU() will only return the information for a single Processor Group.
// Since a single group can only hold 64 logical processors, this
// means when there are more they will be divided into multiple groups.
// For the above reason, procGetActiveProcessorCount is used to get the
// cpu count for all processor groups of the windows node.
// more notes for this issue:
// same issue in moby: https://github.com/moby/moby/issues/38935#issuecomment-744638345
// solution in hcsshim: https://github.com/microsoft/hcsshim/blob/master/internal/processorinfo/processor_count.go
func ProcessorCount() int {
	if amount := getActiveProcessorCount(allProcessorGroups); amount != 0 {
		return int(amount)
	}
	return runtime.NumCPU()
}


func getActiveProcessorCount(groupNumber uint16) int {
	r0, _, _ := syscall.Syscall(procGetActiveProcessorCount.Addr(), 1, uintptr(groupNumber), 0, 0)
	return int(r0)
}


func (p *perfCounterNodeStatsClient) getVersionInfo() (*cadvisorapi.VersionInfo, error) {
	return &cadvisorapi.VersionInfo{
		KernelVersion:      p.nodeInfo.kernelVersion,
		ContainerOsVersion: p.nodeInfo.osImageVersion,
	}, nil
}


func (p *perfCounterNodeStatsClient) getNodeMetrics() (nodeMetrics, error) {
	p.mu.RLock()
	defer p.mu.RUnlock()
	return p.nodeMetrics, nil
}


func (p *perfCounterNodeStatsClient) getNodeInfo() nodeInfo {
	return p.nodeInfo
}


func (p *perfCounterNodeStatsClient) collectMetricsData(cpuCounter, memWorkingSetCounter, memCommittedBytesCounter perfCounter, networkAdapterCounter *networkCounter) {
	cpuValue, err := cpuCounter.getData()
	cpuCores := ProcessorCount()
	if err != nil {
		klog.ErrorS(err, "Unable to get cpu perf counter data")
		return
	}

	memWorkingSetValue, err := memWorkingSetCounter.getData()
	if err != nil {
		klog.ErrorS(err, "Unable to get memWorkingSet perf counter data")
		return
	}

	memCommittedBytesValue, err := memCommittedBytesCounter.getData()
	if err != nil {
		klog.ErrorS(err, "Unable to get memCommittedBytes perf counter data")
		return
	}

	networkAdapterStats, err := networkAdapterCounter.getData()
	if err != nil {
		klog.ErrorS(err, "Unable to get network adapter perf counter data")
		return
	}

	p.mu.Lock()
	defer p.mu.Unlock()
	p.nodeMetrics = nodeMetrics{
		cpuUsageCoreNanoSeconds:   p.convertCPUValue(cpuCores, cpuValue),
		cpuUsageNanoCores:         p.getCPUUsageNanoCores(),
		memoryPrivWorkingSetBytes: memWorkingSetValue,
		memoryCommittedBytes:      memCommittedBytesValue,
		interfaceStats:            networkAdapterStats,
		timeStamp:                 time.Now(),
	}
}


func (p *perfCounterNodeStatsClient) convertCPUValue(cpuCores int, cpuValue uint64) uint64 {
	// This converts perf counter data which is cpu percentage for all cores into nanoseconds.
	// The formula is (cpuPercentage / 100.0) * #cores * 1e+9 (nano seconds). More info here:
	// https://github.com/kubernetes/heapster/issues/650
	newValue := p.nodeMetrics.cpuUsageCoreNanoSeconds + uint64((float64(cpuValue)/100.0)*float64(cpuCores)*1e9)
	return newValue
}


func (p *perfCounterNodeStatsClient) getCPUUsageNanoCores() uint64 {
	cachePeriodSeconds := uint64(defaultCachePeriod / time.Second)
	cpuUsageNanoCores := (p.cpuUsageCoreNanoSecondsCache.latestValue - p.cpuUsageCoreNanoSecondsCache.previousValue) / cachePeriodSeconds
	return cpuUsageNanoCores
}


func getSystemUUID() (string, error)

func getSystemUUID() (string, error) {
	result, err := exec.Command("wmic", "csproduct", "get", "UUID").Output()
	if err != nil {
		return "", err
	}
	fields := strings.Fields(string(result))
	if len(fields) != 2 {
		return "", fmt.Errorf("received unexpected value retrieving vm uuid: %q", string(result))
	}
	return fields[1], nil
}


func getPhysicallyInstalledSystemMemoryBytes() (uint64, error) {
	// We use GlobalMemoryStatusEx instead of GetPhysicallyInstalledSystemMemory
	// on Windows node for the following reasons:
	// 1. GetPhysicallyInstalledSystemMemory retrieves the amount of physically
	// installed RAM from the computer's SMBIOS firmware tables.
	// https://msdn.microsoft.com/en-us/library/windows/desktop/cc300158(v=vs.85).aspx
	// On some VM, it is unable to read data from SMBIOS and fails with ERROR_INVALID_DATA.
	// 2. On Linux node, total physical memory is read from MemTotal in /proc/meminfo.
	// GlobalMemoryStatusEx returns the amount of physical memory that is available
	// for the operating system to use. The amount returned by GlobalMemoryStatusEx
	// is closer in parity with Linux
	// https://www.kernel.org/doc/Documentation/filesystems/proc.txt
	var statex MemoryStatusEx
	statex.Length = uint32(unsafe.Sizeof(statex))
	ret, _, _ := procGlobalMemoryStatusEx.Call(uintptr(unsafe.Pointer(&statex)))

	if ret == 0 {
		return 0, errors.New("unable to read physical memory")
	}

	return statex.TotalPhys, nil
}


func getBootID() (string, error)

func getBootID() (string, error) {
	regKey, err := registry.OpenKey(registry.LOCAL_MACHINE, bootIdRegistry, registry.READ)
	if err != nil {
		return "", err
	}
	defer regKey.Close()
	regValue, _, err := regKey.GetIntegerValue(bootIdKey)
	if err != nil {
		return "", err
	}
	return strconv.FormatUint(regValue, 10), nil
}


func newPerfCounter(counter string) (perfCounter, error) {
	var queryHandle win_pdh.PDH_HQUERY
	var counterHandle win_pdh.PDH_HCOUNTER

	ret := win_pdh.PdhOpenQuery(0, 0, &queryHandle)
	if ret != win_pdh.ERROR_SUCCESS {
		return nil, errors.New("unable to open query through DLL call")
	}

	ret = win_pdh.PdhAddEnglishCounter(queryHandle, counter, 0, &counterHandle)
	if ret != win_pdh.ERROR_SUCCESS {
		return nil, fmt.Errorf("unable to add process counter: %s. Error code is %x", counter, ret)
	}

	ret = win_pdh.PdhCollectQueryData(queryHandle)
	if ret != win_pdh.ERROR_SUCCESS {
		return nil, fmt.Errorf("unable to collect data from counter. Error code is %x", ret)
	}

	return &perfCounterImpl{
		queryHandle:   queryHandle,
		counterHandle: counterHandle,
	}, nil
}


// getData is used for getting data without * in counter name.
func (p *perfCounterImpl) getData() (uint64, error) {
	filledBuf, bufCount, err := p.getQueriedData()
	if err != nil {
		return 0, err
	}

	var data uint64 = 0
	for i := 0; i < int(bufCount); i++ {
		c := filledBuf[i]
		data = uint64(c.FmtValue.DoubleValue)
	}

	return data, nil
}


// getDataList is used for getting data with * in counter name.
func (p *perfCounterImpl) getDataList() (map[string]uint64, error) {
	filledBuf, bufCount, err := p.getQueriedData()
	if err != nil {
		return nil, err
	}

	data := map[string]uint64{}
	for i := 0; i < int(bufCount); i++ {
		c := filledBuf[i]
		value := uint64(c.FmtValue.DoubleValue)
		name := win_pdh.UTF16PtrToString(c.SzName)
		data[name] = value
	}

	return data, nil
}


// getQueriedData is used for getting data using the given query handle.
func (p *perfCounterImpl) getQueriedData() ([]win_pdh.PDH_FMT_COUNTERVALUE_ITEM_DOUBLE, uint32, error) {
	ret := win_pdh.PdhCollectQueryData(p.queryHandle)
	if ret != win_pdh.ERROR_SUCCESS {
		return nil, 0, fmt.Errorf("unable to collect data from counter. Error code is %x", ret)
	}

	var bufSize, bufCount uint32
	var size = uint32(unsafe.Sizeof(win_pdh.PDH_FMT_COUNTERVALUE_ITEM_DOUBLE{}))
	var emptyBuf [1]win_pdh.PDH_FMT_COUNTERVALUE_ITEM_DOUBLE // need at least 1 addressable null ptr.

	ret = win_pdh.PdhGetFormattedCounterArrayDouble(p.counterHandle, &bufSize, &bufCount, &emptyBuf[0])
	if ret != win_pdh.PDH_MORE_DATA {
		return nil, 0, fmt.Errorf("unable to collect data from counter. Error code is %x", ret)
	}

	filledBuf := make([]win_pdh.PDH_FMT_COUNTERVALUE_ITEM_DOUBLE, bufCount*size)
	ret = win_pdh.PdhGetFormattedCounterArrayDouble(p.counterHandle, &bufSize, &bufCount, &filledBuf[0])
	if ret != win_pdh.ERROR_SUCCESS {
		return nil, 0, fmt.Errorf("unable to collect data from counter. Error code is %x", ret)
	}

	return filledBuf, bufCount, nil
}


func GetOSInfo() (*OSInfo, error)

// GetOSInfo reads Windows version information from the registry
func GetOSInfo() (*OSInfo, error) {
	// for log detail
	var keyPrefix string = `regedit:LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion`

	k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows NT\CurrentVersion`, registry.QUERY_VALUE)
	if err != nil {
		return nil, fmt.Errorf("getOSInfo, open Windows %s failed: %w", keyPrefix, err)
	}
	defer k.Close()

	buildNumber, _, err := k.GetStringValue("CurrentBuildNumber")
	if err != nil {
		return nil, fmt.Errorf("getOSInfo, get %s\\CurrentBuildNumber failed: %w", keyPrefix, err)
	}

	majorVersionNumber, _, err := k.GetIntegerValue("CurrentMajorVersionNumber")
	if err != nil {
		return nil, fmt.Errorf("getOSInfo, get %s\\CurrentMajorVersionNumber failed: %w", keyPrefix, err)
	}

	minorVersionNumber, _, err := k.GetIntegerValue("CurrentMinorVersionNumber")
	if err != nil {
		return nil, fmt.Errorf("getOSInfo, get %s\\CurrentMinorVersionNumber failed: %w", keyPrefix, err)
	}

	revision, _, err := k.GetIntegerValue("UBR")
	if err != nil {
		return nil, fmt.Errorf("getOSInfo, get %s\\UBR failed: %w", keyPrefix, err)
	}

	productName, _, err := k.GetStringValue("ProductName")
	if err != nil {
		return nil, fmt.Errorf("getOSInfo, get %s\\ProductName failed: %w", keyPrefix, err)
	}

	return &OSInfo{
		BuildNumber:  buildNumber,
		ProductName:  productName,
		MajorVersion: majorVersionNumber,
		MinorVersion: minorVersionNumber,
		UBR:          revision,
	}, nil
}


// GetPatchVersion returns full OS version with patch
func (o *OSInfo) GetPatchVersion() string {
	return fmt.Sprintf("%d.%d.%s.%d", o.MajorVersion, o.MinorVersion, o.BuildNumber, o.UBR)
}


func (o *OSInfo) GetBuild() string

// GetBuild returns OS version upto build number
func (o *OSInfo) GetBuild() string {
	return fmt.Sprintf("%d.%d.%s", o.MajorVersion, o.MinorVersion, o.BuildNumber)
}


// newClient constructs a Client.
func newClient(statsNodeClient winNodeStatsClient) (Client, error) {
	statsClient := new(StatsClient)
	statsClient.client = statsNodeClient

	err := statsClient.client.startMonitoring()
	if err != nil {
		return nil, err
	}

	return statsClient, nil
}


// WinContainerInfos returns a map of container infos. The map contains node and
// pod level stats. Analogous to cadvisor GetContainerInfoV2 method.
func (c *StatsClient) WinContainerInfos() (map[string]cadvisorapiv2.ContainerInfo, error) {
	infos := make(map[string]cadvisorapiv2.ContainerInfo)
	rootContainerInfo, err := c.createRootContainerInfo()
	if err != nil {
		return nil, err
	}

	infos["/"] = *rootContainerInfo

	return infos, nil
}


// WinMachineInfo returns a cadvisorapi.MachineInfo with details about the
// node machine. Analogous to cadvisor MachineInfo method.
func (c *StatsClient) WinMachineInfo() (*cadvisorapi.MachineInfo, error) {
	return c.client.getMachineInfo()
}


// WinVersionInfo returns a  cadvisorapi.VersionInfo with version info of
// the kernel and docker runtime. Analogous to cadvisor VersionInfo method.
func (c *StatsClient) WinVersionInfo() (*cadvisorapi.VersionInfo, error) {
	return c.client.getVersionInfo()
}


func (c *StatsClient) createRootContainerInfo() (*cadvisorapiv2.ContainerInfo, error) {
	nodeMetrics, err := c.client.getNodeMetrics()
	if err != nil {
		return nil, err
	}

	var stats []*cadvisorapiv2.ContainerStats
	stats = append(stats, &cadvisorapiv2.ContainerStats{
		Timestamp: nodeMetrics.timeStamp,
		Cpu: &cadvisorapi.CpuStats{
			Usage: cadvisorapi.CpuUsage{
				Total: nodeMetrics.cpuUsageCoreNanoSeconds,
			},
		},
		CpuInst: &cadvisorapiv2.CpuInstStats{
			Usage: cadvisorapiv2.CpuInstUsage{
				Total: nodeMetrics.cpuUsageNanoCores,
			},
		},
		Memory: &cadvisorapi.MemoryStats{
			WorkingSet: nodeMetrics.memoryPrivWorkingSetBytes,
			Usage:      nodeMetrics.memoryCommittedBytes,
		},
		Network: &cadvisorapiv2.NetworkStats{
			Interfaces: nodeMetrics.interfaceStats,
		},
	})

	nodeInfo := c.client.getNodeInfo()
	rootInfo := cadvisorapiv2.ContainerInfo{
		Spec: cadvisorapiv2.ContainerSpec{
			CreationTime: nodeInfo.startTime,
			HasCpu:       true,
			HasMemory:    true,
			HasNetwork:   true,
			Memory: cadvisorapiv2.MemorySpec{
				Limit: nodeInfo.memoryPhysicalCapacityBytes,
			},
		},
		Stats: stats,
	}

	return &rootInfo, nil
}


// GetDirFsInfo returns filesystem capacity and usage information.
func (c *StatsClient) GetDirFsInfo(path string) (cadvisorapiv2.FsInfo, error) {
	var freeBytesAvailable, totalNumberOfBytes, totalNumberOfFreeBytes int64
	var err error

	ret, _, err := syscall.Syscall6(
		procGetDiskFreeSpaceEx.Addr(),
		4,
		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
		uintptr(unsafe.Pointer(&freeBytesAvailable)),
		uintptr(unsafe.Pointer(&totalNumberOfBytes)),
		uintptr(unsafe.Pointer(&totalNumberOfFreeBytes)),
		0,
		0,
	)
	if ret == 0 {
		return cadvisorapiv2.FsInfo{}, err
	}

	return cadvisorapiv2.FsInfo{
		Timestamp: time.Now(),
		Capacity:  uint64(totalNumberOfBytes),
		Available: uint64(freeBytesAvailable),
		Usage:     uint64(totalNumberOfBytes - freeBytesAvailable),
	}, nil
}


// NewKubemarkController creates KubemarkController using the provided clients to talk to external
// and kubemark clusters.
func NewKubemarkController(externalClient kubeclient.Interface, externalInformerFactory informers.SharedInformerFactory,
	kubemarkClient kubeclient.Interface, kubemarkNodeInformer informersv1.NodeInformer) (*KubemarkController, error) {
	rcInformer := externalInformerFactory.InformerFor(&apiv1.ReplicationController{}, newReplicationControllerInformer)
	podInformer := externalInformerFactory.InformerFor(&apiv1.Pod{}, newPodInformer)
	controller := &KubemarkController{
		externalCluster: externalCluster{
			rcLister:  listersv1.NewReplicationControllerLister(rcInformer.GetIndexer()),
			rcSynced:  rcInformer.HasSynced,
			podLister: listersv1.NewPodLister(podInformer.GetIndexer()),
			podSynced: podInformer.HasSynced,
			client:    externalClient,
		},
		kubemarkCluster: kubemarkCluster{
			nodeLister:        kubemarkNodeInformer.Lister(),
			nodeSynced:        kubemarkNodeInformer.Informer().HasSynced,
			client:            kubemarkClient,
			nodesToDelete:     make(map[string]bool),
			nodesToDeleteLock: sync.Mutex{},
		},
		rand:                   rand.New(rand.NewSource(time.Now().UnixNano())),
		createNodeQueue:        make(chan string, 1000),
		nodeGroupQueueSize:     make(map[string]int),
		nodeGroupQueueSizeLock: sync.Mutex{},
	}

	kubemarkNodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		UpdateFunc: controller.kubemarkCluster.removeUnneededNodes,
	})

	return controller, nil
}


// WaitForCacheSync waits until all caches in the controller are populated.
func (kubemarkController *KubemarkController) WaitForCacheSync(stopCh chan struct{}) bool {
	return cache.WaitForNamedCacheSync("kubemark", stopCh,
		kubemarkController.externalCluster.rcSynced,
		kubemarkController.externalCluster.podSynced,
		kubemarkController.kubemarkCluster.nodeSynced)
}


// Run populates the node template needed for creation of kubemark nodes and
// starts the worker routine for creating new nodes.
func (kubemarkController *KubemarkController) Run(stopCh chan struct{}) {
	nodeTemplate, err := kubemarkController.getNodeTemplate()
	if err != nil {
		klog.Fatalf("failed to get node template: %s", err)
	}
	kubemarkController.nodeTemplate = nodeTemplate

	go kubemarkController.runNodeCreation(stopCh)
	<-stopCh
}


// GetNodeNamesForNodeGroup returns list of the nodes in the node group.
func (kubemarkController *KubemarkController) GetNodeNamesForNodeGroup(nodeGroup string) ([]string, error) {
	selector := labels.SelectorFromSet(labels.Set{nodeGroupLabel: nodeGroup})
	pods, err := kubemarkController.externalCluster.podLister.List(selector)
	if err != nil {
		return nil, err
	}
	result := make([]string, 0, len(pods))
	for _, pod := range pods {
		result = append(result, pod.ObjectMeta.Name)
	}
	return result, nil
}


// GetNodeGroupSize returns the current size for the node group as observed.
func (kubemarkController *KubemarkController) GetNodeGroupSize(nodeGroup string) (int, error) {
	selector := labels.SelectorFromSet(labels.Set(map[string]string{nodeGroupLabel: nodeGroup}))
	nodes, err := kubemarkController.externalCluster.rcLister.List(selector)
	if err != nil {
		return 0, err
	}
	return len(nodes), nil
}


// GetNodeGroupTargetSize returns the size of the node group as a sum of current
// observed size and number of upcoming nodes.
func (kubemarkController *KubemarkController) GetNodeGroupTargetSize(nodeGroup string) (int, error) {
	kubemarkController.nodeGroupQueueSizeLock.Lock()
	defer kubemarkController.nodeGroupQueueSizeLock.Unlock()
	realSize, err := kubemarkController.GetNodeGroupSize(nodeGroup)
	if err != nil {
		return realSize, err
	}
	return realSize + kubemarkController.nodeGroupQueueSize[nodeGroup], nil
}


// SetNodeGroupSize changes the size of node group by adding or removing nodes.
func (kubemarkController *KubemarkController) SetNodeGroupSize(nodeGroup string, size int) error {
	currSize, err := kubemarkController.GetNodeGroupTargetSize(nodeGroup)
	if err != nil {
		return err
	}
	switch delta := size - currSize; {
	case delta < 0:
		absDelta := -delta
		nodes, err := kubemarkController.GetNodeNamesForNodeGroup(nodeGroup)
		if err != nil {
			return err
		}
		if len(nodes) < absDelta {
			return fmt.Errorf("can't remove %d nodes from %s nodegroup, not enough nodes: %d", absDelta, nodeGroup, len(nodes))
		}
		for i, node := range nodes {
			if i == absDelta {
				return nil
			}
			if err := kubemarkController.RemoveNodeFromNodeGroup(nodeGroup, node); err != nil {
				return err
			}
		}
	case delta > 0:
		kubemarkController.nodeGroupQueueSizeLock.Lock()
		kubemarkController.nodeGroupQueueSize[nodeGroup] += delta
		kubemarkController.nodeGroupQueueSizeLock.Unlock()
		for i := 0; i < delta; i++ {
			kubemarkController.createNodeQueue <- nodeGroup
		}
	}

	return nil
}


// GetNodeGroupForNode returns the name of the node group to which the node
// belongs.
func (kubemarkController *KubemarkController) GetNodeGroupForNode(node string) (string, error) {
	pod := kubemarkController.getPodByName(node)
	if pod == nil {
		return "", fmt.Errorf("node %s does not exist", node)
	}
	nodeGroup, ok := pod.ObjectMeta.Labels[nodeGroupLabel]
	if ok {
		return nodeGroup, nil
	}
	return "", fmt.Errorf("can't find nodegroup for node %s due to missing label %s", node, nodeGroupLabel)
}


func (kubemarkController *KubemarkController) addNodeToNodeGroup(nodeGroup string) error {
	node := kubemarkController.nodeTemplate.DeepCopy()
	node.Name = fmt.Sprintf("%s-%d", nodeGroup, kubemarkController.rand.Int63())
	node.Labels = map[string]string{nodeGroupLabel: nodeGroup, "name": node.Name}
	node.Spec.Template.Labels = node.Labels

	var err error
	for i := 0; i < numRetries; i++ {
		_, err = kubemarkController.externalCluster.client.CoreV1().ReplicationControllers(node.Namespace).Create(context.TODO(), node, metav1.CreateOptions{})
		if err == nil {
			return nil
		}
	}

	return err
}


func (kubemarkController *KubemarkController) RemoveNodeFromNodeGroup(nodeGroup string, node string) error {
	pod := kubemarkController.getPodByName(node)
	if pod == nil {
		klog.Warningf("Can't delete node %s from nodegroup %s. Node does not exist.", node, nodeGroup)
		return nil
	}
	if pod.ObjectMeta.Labels[nodeGroupLabel] != nodeGroup {
		return fmt.Errorf("can't delete node %s from nodegroup %s. Node is not in nodegroup", node, nodeGroup)
	}
	policy := metav1.DeletePropagationForeground
	var err error
	for i := 0; i < numRetries; i++ {
		err = kubemarkController.externalCluster.client.CoreV1().ReplicationControllers(namespaceKubemark).Delete(context.TODO(), pod.ObjectMeta.Labels["name"],
			metav1.DeleteOptions{PropagationPolicy: &policy})
		if err == nil {
			klog.Infof("marking node %s for deletion", node)
			// Mark node for deletion from kubemark cluster.
			// Once it becomes unready after replication controller
			// deletion has been noticed, we will delete it explicitly.
			// This is to cover for the fact that kubemark does not
			// take care of this itself.
			kubemarkController.kubemarkCluster.markNodeForDeletion(node)
			return nil
		}
	}
	return fmt.Errorf("Failed to delete node %s: %v", node, err)
}


func (kubemarkController *KubemarkController) getReplicationControllerByName(name string) *apiv1.ReplicationController {
	rcs, err := kubemarkController.externalCluster.rcLister.List(labels.Everything())
	if err != nil {
		return nil
	}
	for _, rc := range rcs {
		if rc.ObjectMeta.Name == name {
			return rc
		}
	}
	return nil
}


func (kubemarkController *KubemarkController) getPodByName(name string) *apiv1.Pod {
	pods, err := kubemarkController.externalCluster.podLister.List(labels.Everything())
	if err != nil {
		return nil
	}
	for _, pod := range pods {
		if pod.ObjectMeta.Name == name {
			return pod
		}
	}
	return nil
}


func (kubemarkController *KubemarkController) getNodeNameForPod(podName string) (string, error) {
	pods, err := kubemarkController.externalCluster.podLister.List(labels.Everything())
	if err != nil {
		return "", err
	}
	for _, pod := range pods {
		if pod.ObjectMeta.Name == podName {
			return pod.Labels["name"], nil
		}
	}
	return "", fmt.Errorf("pod %s not found", podName)
}


// getNodeTemplate returns the template for hollow node replication controllers
// by looking for an existing hollow node specification. This requires at least
// one kubemark node to be present on startup.
func (kubemarkController *KubemarkController) getNodeTemplate() (*apiv1.ReplicationController, error) {
	podName, err := kubemarkController.kubemarkCluster.getHollowNodeName()
	if err != nil {
		return nil, err
	}
	hollowNodeName, err := kubemarkController.getNodeNameForPod(podName)
	if err != nil {
		return nil, err
	}
	if hollowNode := kubemarkController.getReplicationControllerByName(hollowNodeName); hollowNode != nil {
		nodeTemplate := &apiv1.ReplicationController{
			Spec: apiv1.ReplicationControllerSpec{
				Template: hollowNode.Spec.Template,
			},
		}

		nodeTemplate.Spec.Selector = nil
		nodeTemplate.Namespace = namespaceKubemark
		one := int32(1)
		nodeTemplate.Spec.Replicas = &one

		return nodeTemplate, nil
	}
	return nil, fmt.Errorf("can't get hollow node template")
}


func (kubemarkController *KubemarkController) runNodeCreation(stop <-chan struct{}) {
	for {
		select {
		case nodeGroup := <-kubemarkController.createNodeQueue:
			kubemarkController.nodeGroupQueueSizeLock.Lock()
			err := kubemarkController.addNodeToNodeGroup(nodeGroup)
			if err != nil {
				klog.Errorf("failed to add node to node group %s: %v", nodeGroup, err)
			} else {
				kubemarkController.nodeGroupQueueSize[nodeGroup]--
			}
			kubemarkController.nodeGroupQueueSizeLock.Unlock()
		case <-stop:
			return
		}
	}
}


func (kubemarkCluster *kubemarkCluster) getHollowNodeName() (string, error) {
	selector, _ := labels.Parse(nodeGroupLabel)
	nodes, err := kubemarkCluster.nodeLister.List(selector)
	if err != nil {
		return "", err
	}
	for _, node := range nodes {
		return node.Name, nil
	}
	return "", fmt.Errorf("did not find any hollow nodes in the cluster")
}


func (kubemarkCluster *kubemarkCluster) removeUnneededNodes(oldObj interface{}, newObj interface{}) {
	node, ok := newObj.(*apiv1.Node)
	if !ok {
		return
	}
	for _, condition := range node.Status.Conditions {
		// Delete node if it is in unready state, and it has been
		// explicitly marked for deletion.
		if condition.Type == apiv1.NodeReady && condition.Status != apiv1.ConditionTrue {
			kubemarkCluster.nodesToDeleteLock.Lock()
			defer kubemarkCluster.nodesToDeleteLock.Unlock()
			if kubemarkCluster.nodesToDelete[node.Name] {
				kubemarkCluster.nodesToDelete[node.Name] = false
				if err := kubemarkCluster.client.CoreV1().Nodes().Delete(context.TODO(), node.Name, metav1.DeleteOptions{}); err != nil {
					klog.Errorf("failed to delete node %s from kubemark cluster, err: %v", node.Name, err)
				}
			}
			return
		}
	}
}


func (kubemarkCluster *kubemarkCluster) markNodeForDeletion(name string) {
	kubemarkCluster.nodesToDeleteLock.Lock()
	defer kubemarkCluster.nodesToDeleteLock.Unlock()
	kubemarkCluster.nodesToDelete[name] = true
}


func newReplicationControllerInformer(kubeClient kubeclient.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
	rcListWatch := cache.NewListWatchFromClient(kubeClient.CoreV1().RESTClient(), "replicationcontrollers", namespaceKubemark, fields.Everything())
	return cache.NewSharedIndexInformer(rcListWatch, &apiv1.ReplicationController{}, resyncPeriod, nil)
}


func newPodInformer(kubeClient kubeclient.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
	podListWatch := cache.NewListWatchFromClient(kubeClient.CoreV1().RESTClient(), "pods", namespaceKubemark, fields.Everything())
	return cache.NewSharedIndexInformer(podListWatch, &apiv1.Pod{}, resyncPeriod, nil)
}


func volumePlugins() []volume.VolumePlugin {
	allPlugins := []volume.VolumePlugin{}
	allPlugins = append(allPlugins, emptydir.ProbeVolumePlugins()...)
	allPlugins = append(allPlugins, git_repo.ProbeVolumePlugins()...)
	allPlugins = append(allPlugins, hostpath.FakeProbeVolumePlugins(volume.VolumeConfig{})...)
	allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(volume.VolumeConfig{})...)
	allPlugins = append(allPlugins, secret.ProbeVolumePlugins()...)
	allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...)
	allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
	allPlugins = append(allPlugins, cephfs.ProbeVolumePlugins()...)
	allPlugins = append(allPlugins, downwardapi.ProbeVolumePlugins()...)
	allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
	allPlugins = append(allPlugins, configmap.ProbeVolumePlugins()...)
	allPlugins = append(allPlugins, projected.ProbeVolumePlugins()...)
	allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
	allPlugins = append(allPlugins, local.ProbeVolumePlugins()...)
	allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
	return allPlugins
}


func NewHollowKubelet(

func NewHollowKubelet(
	flags *options.KubeletFlags,
	config *kubeletconfig.KubeletConfiguration,
	client *clientset.Clientset,
	heartbeatClient *clientset.Clientset,
	cadvisorInterface cadvisor.Interface,
	imageService internalapi.ImageManagerService,
	runtimeService internalapi.RuntimeService,
	containerManager cm.ContainerManager) *HollowKubelet {
	d := &kubelet.Dependencies{
		KubeClient:               client,
		HeartbeatClient:          heartbeatClient,
		ProbeManager:             probetest.FakeManager{},
		RemoteRuntimeService:     runtimeService,
		RemoteImageService:       imageService,
		CAdvisorInterface:        cadvisorInterface,
		Cloud:                    nil,
		OSInterface:              &containertest.FakeOS{},
		ContainerManager:         containerManager,
		VolumePlugins:            volumePlugins(),
		TLSOptions:               nil,
		OOMAdjuster:              oom.NewFakeOOMAdjuster(),
		Mounter:                  &mount.FakeMounter{},
		Subpather:                &subpath.FakeSubpath{},
		HostUtil:                 hostutil.NewFakeHostUtil(nil),
		PodStartupLatencyTracker: kubeletutil.NewPodStartupLatencyTracker(),
		TracerProvider:           trace.NewNoopTracerProvider(),
		Recorder:                 &record.FakeRecorder{}, // With real recorder we attempt to read /dev/kmsg.
	}

	return &HollowKubelet{
		KubeletFlags:         flags,
		KubeletConfiguration: config,
		KubeletDeps:          d,
	}
}


func (hk *HollowKubelet) Run()

// Starts this HollowKubelet and blocks.
func (hk *HollowKubelet) Run() {
	if err := kubeletapp.RunKubelet(&options.KubeletServer{
		KubeletFlags:         *hk.KubeletFlags,
		KubeletConfiguration: *hk.KubeletConfiguration,
	}, hk.KubeletDeps, false); err != nil {
		klog.Fatalf("Failed to run HollowKubelet: %v. Exiting.", err)
	}
	select {}
}


// Builds a KubeletConfiguration for the HollowKubelet, ensuring that the
// usual defaults are applied for fields we do not override.
func GetHollowKubeletConfig(opt *HollowKubeletOptions) (*options.KubeletFlags, *kubeletconfig.KubeletConfiguration) {
	testRootDir := utils.MakeTempDirOrDie("hollow-kubelet.", "")
	podFilePath := utils.MakeTempDirOrDie("static-pods", testRootDir)
	klog.Infof("Using %s as root dir for hollow-kubelet", testRootDir)

	// Flags struct
	f := options.NewKubeletFlags()
	f.RootDirectory = testRootDir
	f.HostnameOverride = opt.NodeName
	f.MinimumGCAge = metav1.Duration{Duration: 1 * time.Minute}
	f.MaxContainerCount = 100
	f.MaxPerPodContainerCount = 2
	f.NodeLabels = opt.NodeLabels
	f.RegisterSchedulable = true

	// Config struct
	c, err := options.NewKubeletConfiguration()
	if err != nil {
		panic(err)
	}

	c.ImageServiceEndpoint = "unix:///run/containerd/containerd.sock"
	c.StaticPodURL = ""
	c.EnableServer = true
	c.Address = "0.0.0.0" /* bind address */
	c.Port = int32(opt.KubeletPort)
	c.ReadOnlyPort = int32(opt.KubeletReadOnlyPort)
	c.StaticPodPath = podFilePath
	c.FileCheckFrequency.Duration = 20 * time.Second
	c.HTTPCheckFrequency.Duration = 20 * time.Second
	c.NodeStatusUpdateFrequency.Duration = 10 * time.Second
	c.NodeStatusReportFrequency.Duration = 5 * time.Minute
	c.SyncFrequency.Duration = 10 * time.Second
	c.EvictionPressureTransitionPeriod.Duration = 5 * time.Minute
	c.MaxPods = int32(opt.MaxPods)
	c.PodsPerCore = int32(opt.PodsPerCore)
	c.ClusterDNS = []string{}
	c.ImageGCHighThresholdPercent = 90
	c.ImageGCLowThresholdPercent = 80
	c.ProviderID = fmt.Sprintf("kubemark://%v", opt.NodeName)
	c.VolumeStatsAggPeriod.Duration = time.Minute
	c.CgroupRoot = ""
	c.CPUCFSQuota = true
	c.EnableControllerAttachDetach = false
	c.EnableDebuggingHandlers = true
	c.CgroupsPerQOS = false
	// hairpin-veth is used to allow hairpin packets. Note that this deviates from
	// what the "real" kubelet currently does, because there's no way to
	// set promiscuous mode on docker0.
	c.HairpinMode = kubeletconfig.HairpinVeth
	c.MaxOpenFiles = 1024
	c.RegistryBurst = 10
	c.RegistryPullQPS = 5.0
	c.ResolverConfig = kubetypes.ResolvConfDefault
	c.KubeletCgroups = "/kubelet"
	c.SerializeImagePulls = true
	c.SystemCgroups = ""
	c.ProtectKernelDefaults = false
	c.RegisterWithTaints = opt.RegisterWithTaints
	c.RegisterNode = true
	c.LocalStorageCapacityIsolation = true

	return f, c
}


func (*FakeProxier) SyncLoop()

func (*FakeProxier) SyncLoop() {
	select {}
}


func (*FakeProxier) OnServiceAdd(service *v1.Service)                                 {}


func (*FakeProxier) OnServiceUpdate(oldService, service *v1.Service)                  {}


func (*FakeProxier) OnServiceDelete(service *v1.Service)                              {}


func (*FakeProxier) OnServiceSynced()                                                 {}


func (*FakeProxier) OnEndpointSliceAdd(slice *discoveryv1.EndpointSlice)              {}


func (*FakeProxier) OnEndpointSliceUpdate(oldSlice, slice *discoveryv1.EndpointSlice) {}


func (*FakeProxier) OnEndpointSliceDelete(slice *discoveryv1.EndpointSlice)           {}


func (*FakeProxier) OnEndpointSlicesSynced()                                          {}


func NewHollowProxyOrDie(

func NewHollowProxyOrDie(
	nodeName string,
	client clientset.Interface,
	eventClient v1core.EventsGetter,
	iptInterface utiliptables.Interface,
	sysctl utilsysctl.Interface,
	execer utilexec.Interface,
	broadcaster events.EventBroadcaster,
	recorder events.EventRecorder,
	useRealProxier bool,
	proxierSyncPeriod time.Duration,
	proxierMinSyncPeriod time.Duration,
) (*HollowProxy, error) {
	// Create proxier and service/endpoint handlers.
	var proxier proxy.Provider
	var err error

	if useRealProxier {
		nodeIP := utilnode.GetNodeIP(client, nodeName)
		if nodeIP == nil {
			klog.InfoS("Can't determine this node's IP, assuming 127.0.0.1")
			nodeIP = netutils.ParseIPSloppy("127.0.0.1")
		}
		family := v1.IPv4Protocol
		if iptInterface.IsIPv6() {
			family = v1.IPv6Protocol
		}
		// Real proxier with fake iptables, sysctl, etc underneath it.
		//var err error
		proxier, err = iptables.NewProxier(
			family,
			iptInterface,
			sysctl,
			execer,
			proxierSyncPeriod,
			proxierMinSyncPeriod,
			false,
			false,
			0,
			proxyutiliptables.NewNoOpLocalDetector(),
			nodeName,
			nodeIP,
			recorder,
			nil,
			[]string{},
		)
		if err != nil {
			return nil, fmt.Errorf("unable to create proxier: %v", err)
		}
	} else {
		proxier = &FakeProxier{}
	}

	// Create a Hollow Proxy instance.
	nodeRef := &v1.ObjectReference{
		Kind:      "Node",
		Name:      nodeName,
		UID:       types.UID(nodeName),
		Namespace: "",
	}
	return &HollowProxy{
		ProxyServer: &proxyapp.ProxyServer{
			Client:           client,
			EventClient:      eventClient,
			IptInterface:     iptInterface,
			Proxier:          proxier,
			Broadcaster:      broadcaster,
			Recorder:         recorder,
			ProxyMode:        "fake",
			NodeRef:          nodeRef,
			OOMScoreAdj:      utilpointer.Int32Ptr(0),
			ConfigSyncPeriod: 30 * time.Second,
		},
	}, nil
}


func (hp *HollowProxy) Run() error

func (hp *HollowProxy) Run() error {
	if err := hp.ProxyServer.Run(); err != nil {
		return fmt.Errorf("Error while running proxy: %w", err)
	}
	return nil
}


// PrintObj implements ResourcePrinter
func (fn ResourcePrinterFunc) PrintObj(obj runtime.Object, w io.Writer) error {
	return fn(obj, w)
}