validateKubeConfig

// validateKubeConfig check if the kubeconfig file exist and has the expected CA and server URL
func validateKubeConfig(outDir, filename string, config *clientcmdapi.Config) error {
	kubeConfigFilePath := filepath.Join(outDir, filename)

	if _, err := os.Stat(kubeConfigFilePath); err != nil {
		return err
	}

	// The kubeconfig already exists, let's check if it has got the same CA and server URL
	currentConfig, err := clientcmd.LoadFromFile(kubeConfigFilePath)
	if err != nil {
		return errors.Wrapf(err, "failed to load kubeconfig file %s that already exists on disk", kubeConfigFilePath)
	}

	expectedCtx, exists := config.Contexts[config.CurrentContext]
	if !exists {
		return errors.Errorf("failed to find expected context %s", config.CurrentContext)
	}
	expectedCluster := expectedCtx.Cluster
	currentCtx, exists := currentConfig.Contexts[currentConfig.CurrentContext]
	if !exists {
		return errors.Errorf("failed to find CurrentContext in Contexts of the kubeconfig file %s", kubeConfigFilePath)
	}
	currentCluster := currentCtx.Cluster
	if currentConfig.Clusters[currentCluster] == nil {
		return errors.Errorf("failed to find the given CurrentContext Cluster in Clusters of the kubeconfig file %s", kubeConfigFilePath)
	}

	// Make sure the compared CAs are whitespace-trimmed. The function clientcmd.LoadFromFile() just decodes
	// the base64 CA and places it raw in the v1.Config object. In case the user has extra whitespace
	// in the CA they used to create a kubeconfig this comparison to a generated v1.Config will otherwise fail.
	caCurrent := bytes.TrimSpace(currentConfig.Clusters[currentCluster].CertificateAuthorityData)
	if len(caCurrent) == 0 {
		// fallback to load CA cert data from external CA file
		clusterCAFilePath := currentConfig.Clusters[currentCluster].CertificateAuthority
		if len(clusterCAFilePath) > 0 {
			clusterCABytes, err := os.ReadFile(clusterCAFilePath)
			if err != nil {
				klog.Warningf("failed to load CA cert from %q for kubeconfig %q, %v", clusterCAFilePath, kubeConfigFilePath, err)
			} else {
				caCurrent = bytes.TrimSpace(clusterCABytes)
			}
		}
	}
	caExpected := bytes.TrimSpace(config.Clusters[expectedCluster].CertificateAuthorityData)

	// If the current CA cert on disk doesn't match the expected CA cert, error out because we have a file, but it's stale
	if !bytes.Equal(caCurrent, caExpected) {
		return errors.Errorf("a kubeconfig file %q exists already but has got the wrong CA cert", kubeConfigFilePath)
	}
	// If the current API Server location on disk doesn't match the expected API server, show a warning
	if currentConfig.Clusters[currentCluster].Server != config.Clusters[expectedCluster].Server {
		klog.Warningf("a kubeconfig file %q exists already but has an unexpected API Server URL: expected: %s, got: %s",
			kubeConfigFilePath, config.Clusters[expectedCluster].Server, currentConfig.Clusters[currentCluster].Server)
	}

	return nil
}


createKubeConfigFileIfNotExists

// createKubeConfigFileIfNotExists saves the KubeConfig object into a file if there isn't any file at the given path.
// If there already is a kubeconfig file at the given path; kubeadm tries to load it and check if the values in the
// existing and the expected config equals. If they do; kubeadm will just skip writing the file as it's up-to-date,
// but if a file exists but has old content or isn't a kubeconfig file, this function returns an error.
func createKubeConfigFileIfNotExists(outDir, filename string, config *clientcmdapi.Config) error {
	kubeConfigFilePath := filepath.Join(outDir, filename)

	err := validateKubeConfig(outDir, filename, config)
	if err != nil {
		// Check if the file exist, and if it doesn't, just write it to disk
		if !os.IsNotExist(err) {
			return err
		}
		fmt.Printf("[kubeconfig] Writing %q kubeconfig file\n", filename)
		err = kubeconfigutil.WriteToDisk(kubeConfigFilePath, config)
		return errors.Wrapf(err, "failed to save kubeconfig file %q on disk", kubeConfigFilePath)
	}
	// kubeadm doesn't validate the existing kubeconfig file more than this (kubeadm trusts the client certs to be valid)
	// Basically, if we find a kubeconfig file with the same path; the same CA cert and the same server URL;
	// kubeadm thinks those files are equal and doesn't bother writing a new file
	fmt.Printf("[kubeconfig] Using existing kubeconfig file: %q\n", kubeConfigFilePath)

	return nil
}


WriteKubeConfigWithClientCert

// WriteKubeConfigWithClientCert writes a kubeconfig file - with a client certificate as authentication info  - to the given writer.
func WriteKubeConfigWithClientCert(out io.Writer, cfg *kubeadmapi.InitConfiguration, clientName string, organizations []string, notAfter *time.Time) error {

	// creates the KubeConfigSpecs, actualized for the current InitConfiguration
	caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
	if err != nil {
		return errors.Wrap(err, "couldn't create a kubeconfig; the CA files couldn't be loaded")
	}
	// Validate period
	certsphase.CheckCertificatePeriodValidity(kubeadmconstants.CACertAndKeyBaseName, caCert)

	controlPlaneEndpoint, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint)
	if err != nil {
		return err
	}

	spec := &kubeConfigSpec{
		ClientName: clientName,
		APIServer:  controlPlaneEndpoint,
		CACert:     caCert,
		ClientCertAuth: &clientCertAuth{
			CAKey:         caKey,
			Organizations: organizations,
		},
	}

	return writeKubeConfigFromSpec(out, spec, cfg.ClusterName, notAfter)
}


WriteKubeConfigWithToken

// WriteKubeConfigWithToken writes a kubeconfig file - with a token as client authentication info - to the given writer.
func WriteKubeConfigWithToken(out io.Writer, cfg *kubeadmapi.InitConfiguration, clientName, token string, notAfter *time.Time) error {

	// creates the KubeConfigSpecs, actualized for the current InitConfiguration
	caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
	if err != nil {
		return errors.Wrap(err, "couldn't create a kubeconfig; the CA files couldn't be loaded")
	}
	// Validate period
	certsphase.CheckCertificatePeriodValidity(kubeadmconstants.CACertAndKeyBaseName, caCert)

	controlPlaneEndpoint, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint)
	if err != nil {
		return err
	}

	spec := &kubeConfigSpec{
		ClientName: clientName,
		APIServer:  controlPlaneEndpoint,
		CACert:     caCert,
		TokenAuth: &tokenAuth{
			Token: token,
		},
	}

	return writeKubeConfigFromSpec(out, spec, cfg.ClusterName, notAfter)
}


writeKubeConfigFromSpec

// writeKubeConfigFromSpec creates a kubeconfig object from a kubeConfigSpec and writes it to the given writer.
func writeKubeConfigFromSpec(out io.Writer, spec *kubeConfigSpec, clustername string, notAfter *time.Time) error {

	// builds the KubeConfig object
	config, err := buildKubeConfigFromSpec(spec, clustername, notAfter)
	if err != nil {
		return err
	}

	// writes the kubeconfig to disk if it not exists
	configBytes, err := clientcmd.Write(*config)
	if err != nil {
		return errors.Wrap(err, "failure while serializing admin kubeconfig")
	}

	fmt.Fprintln(out, string(configBytes))
	return nil
}


ValidateKubeconfigsForExternalCA

// ValidateKubeconfigsForExternalCA check if the kubeconfig file exist and has the expected CA and server URL using kubeadmapi.InitConfiguration.
func ValidateKubeconfigsForExternalCA(outDir string, cfg *kubeadmapi.InitConfiguration) error {
	// Creates a kubeconfig file with the target CA and server URL
	// to be used as a input for validating user provided kubeconfig files
	caCert, err := pkiutil.TryLoadCertFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
	if err != nil {
		return errors.Wrapf(err, "the CA file couldn't be loaded")
	}
	// Validate period
	certsphase.CheckCertificatePeriodValidity(kubeadmconstants.CACertAndKeyBaseName, caCert)

	// validate user provided kubeconfig files for the scheduler and controller-manager
	localAPIEndpoint, err := kubeadmutil.GetLocalAPIEndpoint(&cfg.LocalAPIEndpoint)
	if err != nil {
		return err
	}

	validationConfigLocal := kubeconfigutil.CreateBasic(localAPIEndpoint, "dummy", "dummy", pkiutil.EncodeCertPEM(caCert))
	kubeConfigFileNamesLocal := []string{
		kubeadmconstants.ControllerManagerKubeConfigFileName,
		kubeadmconstants.SchedulerKubeConfigFileName,
	}

	for _, kubeConfigFileName := range kubeConfigFileNamesLocal {
		if err = validateKubeConfig(outDir, kubeConfigFileName, validationConfigLocal); err != nil {
			return errors.Wrapf(err, "the %s file does not exists or it is not valid", kubeConfigFileName)
		}
	}

	// validate user provided kubeconfig files for the kubelet and admin
	controlPlaneEndpoint, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint)
	if err != nil {
		return err
	}

	validationConfigCPE := kubeconfigutil.CreateBasic(controlPlaneEndpoint, "dummy", "dummy", pkiutil.EncodeCertPEM(caCert))
	kubeConfigFileNamesCPE := []string{
		kubeadmconstants.AdminKubeConfigFileName,
		kubeadmconstants.KubeletKubeConfigFileName,
	}

	for _, kubeConfigFileName := range kubeConfigFileNamesCPE {
		if err = validateKubeConfig(outDir, kubeConfigFileName, validationConfigCPE); err != nil {
			return errors.Wrapf(err, "the %s file does not exists or it is not valid", kubeConfigFileName)
		}
	}

	return nil
}


getKubeConfigSpecsBase

func getKubeConfigSpecsBase(cfg *kubeadmapi.InitConfiguration) (map[string]*kubeConfigSpec, error) {
	controlPlaneEndpoint, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint)
	if err != nil {
		return nil, err
	}
	localAPIEndpoint, err := kubeadmutil.GetLocalAPIEndpoint(&cfg.LocalAPIEndpoint)
	if err != nil {
		return nil, err
	}

	return map[string]*kubeConfigSpec{
		kubeadmconstants.AdminKubeConfigFileName: {
			APIServer:  controlPlaneEndpoint,
			ClientName: "kubernetes-admin",
			ClientCertAuth: &clientCertAuth{
				Organizations: []string{kubeadmconstants.SystemPrivilegedGroup},
			},
		},
		kubeadmconstants.KubeletKubeConfigFileName: {
			APIServer:  controlPlaneEndpoint,
			ClientName: fmt.Sprintf("%s%s", kubeadmconstants.NodesUserPrefix, cfg.NodeRegistration.Name),
			ClientCertAuth: &clientCertAuth{
				Organizations: []string{kubeadmconstants.NodesGroup},
			},
		},
		kubeadmconstants.ControllerManagerKubeConfigFileName: {
			APIServer:      localAPIEndpoint,
			ClientName:     kubeadmconstants.ControllerManagerUser,
			ClientCertAuth: &clientCertAuth{},
		},
		kubeadmconstants.SchedulerKubeConfigFileName: {
			APIServer:      localAPIEndpoint,
			ClientName:     kubeadmconstants.SchedulerUser,
			ClientCertAuth: &clientCertAuth{},
		},
	}, nil
}


createKubeConfigAndCSR

func createKubeConfigAndCSR(kubeConfigDir string, kubeadmConfig *kubeadmapi.InitConfiguration, name string, spec *kubeConfigSpec) error {
	if kubeConfigDir == "" {
		return errors.Errorf("%s: kubeConfigDir was empty", errInvalid)
	}
	if kubeadmConfig == nil {
		return errors.Errorf("%s: kubeadmConfig was nil", errInvalid)
	}
	if name == "" {
		return errors.Errorf("%s: name was empty", errInvalid)
	}
	if spec == nil {
		return errors.Errorf("%s: spec was nil", errInvalid)
	}
	kubeConfigPath := filepath.Join(kubeConfigDir, name)
	if _, err := os.Stat(kubeConfigPath); err == nil {
		return errors.Errorf("%s: kube config: %s", errExist, kubeConfigPath)
	} else if !os.IsNotExist(err) {
		return errors.Wrapf(err, "unexpected error while checking if file exists: %s", kubeConfigPath)
	}
	if pkiutil.CSROrKeyExist(kubeConfigDir, name) {
		return errors.Errorf("%s: csr: %s", errExist, kubeConfigPath)
	}

	clientCertConfig := newClientCertConfigFromKubeConfigSpec(spec, nil)

	clientKey, err := pkiutil.NewPrivateKey(clientCertConfig.PublicKeyAlgorithm)
	if err != nil {
		return err
	}
	clientCSR, err := pkiutil.NewCSR(clientCertConfig, clientKey)
	if err != nil {
		return err
	}

	encodedClientKey, err := keyutil.MarshalPrivateKeyToPEM(clientKey)
	if err != nil {
		return err
	}

	var (
		emptyCACert     []byte
		emptyClientCert []byte
	)

	// create a kubeconfig with the client certs
	config := kubeconfigutil.CreateWithCerts(
		spec.APIServer,
		kubeadmConfig.ClusterName,
		spec.ClientName,
		emptyCACert,
		encodedClientKey,
		emptyClientCert,
	)

	if err := kubeconfigutil.WriteToDisk(kubeConfigPath, config); err != nil {
		return err
	}
	// Write CSR to disk
	if err := pkiutil.WriteCSR(kubeConfigDir, name, clientCSR); err != nil {
		return err
	}
	return nil
}


CreateDefaultKubeConfigsAndCSRFiles

// CreateDefaultKubeConfigsAndCSRFiles is used in ExternalCA mode to create
// kubeconfig files and adjacent CSR files.
func CreateDefaultKubeConfigsAndCSRFiles(out io.Writer, kubeConfigDir string, kubeadmConfig *kubeadmapi.InitConfiguration) error {
	kubeConfigs, err := getKubeConfigSpecsBase(kubeadmConfig)
	if err != nil {
		return err
	}
	if out != nil {
		fmt.Fprintf(out, "generating keys and CSRs in %s\n", kubeConfigDir)
	}
	for name, spec := range kubeConfigs {
		if err := createKubeConfigAndCSR(kubeConfigDir, kubeadmConfig, name, spec); err != nil {
			return err
		}
		if out != nil {
			fmt.Fprintf(out, "  %s\n", name)
		}
	}
	return nil
}


WriteConfigToDisk

// WriteConfigToDisk writes the kubelet config object down to a file
// Used at "kubeadm init" and "kubeadm upgrade" time
func WriteConfigToDisk(cfg *kubeadmapi.ClusterConfiguration, kubeletDir, patchesDir string, output io.Writer) error {
	kubeletCfg, ok := cfg.ComponentConfigs[componentconfigs.KubeletGroup]
	if !ok {
		return errors.New("no kubelet component config found")
	}

	if err := kubeletCfg.Mutate(); err != nil {
		return err
	}

	kubeletBytes, err := kubeletCfg.Marshal()
	if err != nil {
		return err
	}

	// Apply patches to the KubeletConfiguration
	if len(patchesDir) != 0 {
		kubeletBytes, err = applyKubeletConfigPatches(kubeletBytes, patchesDir, output)
		if err != nil {
			return errors.Wrap(err, "could not apply patches to the KubeletConfiguration")
		}
	}

	return writeConfigBytesToDisk(kubeletBytes, kubeletDir)
}


CreateConfigMap

// CreateConfigMap creates a ConfigMap with the generic kubelet configuration.
// Used at "kubeadm init" and "kubeadm upgrade" time
func CreateConfigMap(cfg *kubeadmapi.ClusterConfiguration, patchesDir string, client clientset.Interface) error {
	kubeletCfg, ok := cfg.ComponentConfigs[componentconfigs.KubeletGroup]
	if !ok {
		return errors.New("no kubelet component config found in the active component config set")
	}

	kubeletBytes, err := kubeletCfg.Marshal()
	if err != nil {
		return err
	}

	// Apply patches to the KubeletConfiguration
	if len(patchesDir) != 0 {
		kubeletBytes, err = applyKubeletConfigPatches(kubeletBytes, patchesDir, os.Stdout)
		if err != nil {
			return errors.Wrap(err, "could not apply patches to the KubeletConfiguration")
		}
	}

	configMapName := kubeadmconstants.KubeletBaseConfigurationConfigMap
	fmt.Printf("[kubelet] Creating a ConfigMap %q in namespace %s with the configuration for the kubelets in the cluster\n", configMapName, metav1.NamespaceSystem)

	configMap := &v1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Name:      configMapName,
			Namespace: metav1.NamespaceSystem,
		},
		Data: map[string]string{
			kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(kubeletBytes),
		},
	}

	if !kubeletCfg.IsUserSupplied() {
		componentconfigs.SignConfigMap(configMap)
	}

	if err := apiclient.CreateOrUpdateConfigMap(client, configMap); err != nil {
		return err
	}

	if err := createConfigMapRBACRules(client); err != nil {
		return errors.Wrap(err, "error creating kubelet configuration configmap RBAC rules")
	}
	return nil
}


createConfigMapRBACRules

// createConfigMapRBACRules creates the RBAC rules for exposing the base kubelet ConfigMap in the kube-system namespace to unauthenticated users
func createConfigMapRBACRules(client clientset.Interface) error {
	if err := apiclient.CreateOrUpdateRole(client, &rbac.Role{
		ObjectMeta: metav1.ObjectMeta{
			Name:      kubeadmconstants.KubeletBaseConfigMapRole,
			Namespace: metav1.NamespaceSystem,
		},
		Rules: []rbac.PolicyRule{
			{
				Verbs:         []string{"get"},
				APIGroups:     []string{""},
				Resources:     []string{"configmaps"},
				ResourceNames: []string{kubeadmconstants.KubeletBaseConfigurationConfigMap},
			},
		},
	}); err != nil {
		return err
	}

	return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{
		ObjectMeta: metav1.ObjectMeta{
			Name:      kubeadmconstants.KubeletBaseConfigMapRole,
			Namespace: metav1.NamespaceSystem,
		},
		RoleRef: rbac.RoleRef{
			APIGroup: rbac.GroupName,
			Kind:     "Role",
			Name:     kubeadmconstants.KubeletBaseConfigMapRole,
		},
		Subjects: []rbac.Subject{
			{
				Kind: rbac.GroupKind,
				Name: kubeadmconstants.NodesGroup,
			},
			{
				Kind: rbac.GroupKind,
				Name: kubeadmconstants.NodeBootstrapTokenAuthGroup,
			},
		},
	})
}


writeConfigBytesToDisk

// writeConfigBytesToDisk writes a byte slice down to disk at the specific location of the kubelet config file
func writeConfigBytesToDisk(b []byte, kubeletDir string) error {
	configFile := filepath.Join(kubeletDir, kubeadmconstants.KubeletConfigurationFileName)
	fmt.Printf("[kubelet-start] Writing kubelet configuration to file %q\n", configFile)

	// creates target folder if not already exists
	if err := os.MkdirAll(kubeletDir, 0700); err != nil {
		return errors.Wrapf(err, "failed to create directory %q", kubeletDir)
	}

	if err := os.WriteFile(configFile, b, 0644); err != nil {
		return errors.Wrapf(err, "failed to write kubelet configuration to the file %q", configFile)
	}
	return nil
}


applyKubeletConfigPatches

// applyKubeletConfigPatches reads patches from a directory and applies them over the input kubeletBytes
func applyKubeletConfigPatches(kubeletBytes []byte, patchesDir string, output io.Writer) ([]byte, error) {
	patchManager, err := patches.GetPatchManagerForPath(patchesDir, patches.KnownTargets(), output)
	if err != nil {
		return nil, err
	}

	patchTarget := &patches.PatchTarget{
		Name:                      patches.KubeletConfiguration,
		StrategicMergePatchObject: kubeletconfig.KubeletConfiguration{},
		Data:                      kubeletBytes,
	}
	if err := patchManager.ApplyPatchesToTarget(patchTarget); err != nil {
		return nil, err
	}

	kubeletBytes, err = yaml.JSONToYAML(patchTarget.Data)
	if err != nil {
		return nil, err
	}
	return kubeletBytes, nil
}


GetNodeNameAndHostname

// GetNodeNameAndHostname obtains the name for this Node using the following precedence
// (from lower to higher):
// - actual hostname
// - NodeRegistrationOptions.Name (same as "--node-name" passed to "kubeadm init/join")
// - "hostname-override" flag in NodeRegistrationOptions.KubeletExtraArgs
// It also returns the hostname or an error if getting the hostname failed.
func GetNodeNameAndHostname(cfg *kubeadmapi.NodeRegistrationOptions) (string, string, error) {
	hostname, err := nodeutil.GetHostname("")
	nodeName := hostname
	if cfg.Name != "" {
		nodeName = cfg.Name
	}
	if name, ok := cfg.KubeletExtraArgs["hostname-override"]; ok {
		nodeName = name
	}
	return nodeName, hostname, err
}


WriteKubeletDynamicEnvFile

// WriteKubeletDynamicEnvFile writes an environment file with dynamic flags to the kubelet.
// Used at "kubeadm init" and "kubeadm join" time.
func WriteKubeletDynamicEnvFile(cfg *kubeadmapi.ClusterConfiguration, nodeReg *kubeadmapi.NodeRegistrationOptions, registerTaintsUsingFlags bool, kubeletDir string) error {
	flagOpts := kubeletFlagsOpts{
		nodeRegOpts:              nodeReg,
		pauseImage:               images.GetPauseImage(cfg),
		registerTaintsUsingFlags: registerTaintsUsingFlags,
	}
	stringMap := buildKubeletArgMap(flagOpts)
	argList := kubeadmutil.BuildArgumentListFromMap(stringMap, nodeReg.KubeletExtraArgs)
	envFileContent := fmt.Sprintf("%s=%q\n", constants.KubeletEnvFileVariableName, strings.Join(argList, " "))

	return writeKubeletFlagBytesToDisk([]byte(envFileContent), kubeletDir)
}


buildKubeletArgMapCommon

// buildKubeletArgMapCommon takes a kubeletFlagsOpts object and builds based on that a string-string map with flags
// that are common to both Linux and Windows
func buildKubeletArgMapCommon(opts kubeletFlagsOpts) map[string]string {
	kubeletFlags := map[string]string{}
	kubeletFlags["container-runtime-endpoint"] = opts.nodeRegOpts.CRISocket

	// This flag passes the pod infra container image (e.g. "pause" image) to the kubelet
	// and prevents its garbage collection
	if opts.pauseImage != "" {
		kubeletFlags["pod-infra-container-image"] = opts.pauseImage
	}

	if opts.registerTaintsUsingFlags && opts.nodeRegOpts.Taints != nil && len(opts.nodeRegOpts.Taints) > 0 {
		taintStrs := []string{}
		for _, taint := range opts.nodeRegOpts.Taints {
			taintStrs = append(taintStrs, taint.ToString())
		}

		kubeletFlags["register-with-taints"] = strings.Join(taintStrs, ",")
	}

	// Pass the "--hostname-override" flag to the kubelet only if it's different from the hostname
	nodeName, hostname, err := GetNodeNameAndHostname(opts.nodeRegOpts)
	if err != nil {
		klog.Warning(err)
	}
	if nodeName != hostname {
		klog.V(1).Infof("setting kubelet hostname-override to %q", nodeName)
		kubeletFlags["hostname-override"] = nodeName
	}

	return kubeletFlags
}


writeKubeletFlagBytesToDisk

// writeKubeletFlagBytesToDisk writes a byte slice down to disk at the specific location of the kubelet flag overrides file
func writeKubeletFlagBytesToDisk(b []byte, kubeletDir string) error {
	kubeletEnvFilePath := filepath.Join(kubeletDir, constants.KubeletEnvFileName)
	fmt.Printf("[kubelet-start] Writing kubelet environment file with flags to file %q\n", kubeletEnvFilePath)

	// creates target folder if not already exists
	if err := os.MkdirAll(kubeletDir, 0700); err != nil {
		return errors.Wrapf(err, "failed to create directory %q", kubeletDir)
	}
	if err := os.WriteFile(kubeletEnvFilePath, b, 0644); err != nil {
		return errors.Wrapf(err, "failed to write kubelet configuration to the file %q", kubeletEnvFilePath)
	}
	return nil
}


buildKubeletArgMap

// buildKubeletArgMap takes a kubeletFlagsOpts object and builds based on that a string-string map with flags
// that should be given to the local kubelet daemon.
func buildKubeletArgMap(opts kubeletFlagsOpts) map[string]string {
	return buildKubeletArgMapCommon(opts)
}


TryStartKubelet

// TryStartKubelet attempts to bring up kubelet service
func TryStartKubelet() {
	// If we notice that the kubelet service is inactive, try to start it
	initSystem, err := initsystem.GetInitSystem()
	if err != nil {
		fmt.Println("[kubelet-start] No supported init system detected, won't make sure the kubelet is running properly.")
		return
	}

	if !initSystem.ServiceExists(kubeadmconstants.Kubelet) {
		fmt.Println("[kubelet-start] Couldn't detect a kubelet service, can't make sure the kubelet is running properly.")
	}

	// This runs "systemctl daemon-reload && systemctl restart kubelet"
	if err := initSystem.ServiceRestart(kubeadmconstants.Kubelet); err != nil {
		klog.Warningf("[kubelet-start] WARNING: unable to start the kubelet service: [%v]\n", err)
		fmt.Printf("[kubelet-start] Please ensure kubelet is reloaded and running manually.\n")
	}
}


TryStopKubelet

// TryStopKubelet attempts to bring down the kubelet service momentarily
func TryStopKubelet() {
	// If we notice that the kubelet service is inactive, try to start it
	initSystem, err := initsystem.GetInitSystem()
	if err != nil {
		fmt.Println("[kubelet-start] No supported init system detected, won't make sure the kubelet not running for a short period of time while setting up configuration for it.")
		return
	}

	if !initSystem.ServiceExists(kubeadmconstants.Kubelet) {
		fmt.Println("[kubelet-start] Couldn't detect a kubelet service, can't make sure the kubelet not running for a short period of time while setting up configuration for it.")
	}

	// This runs "systemctl daemon-reload && systemctl stop kubelet"
	if err := initSystem.ServiceStop(kubeadmconstants.Kubelet); err != nil {
		klog.Warningf("[kubelet-start] WARNING: unable to stop the kubelet service momentarily: [%v]\n", err)
	}
}


TryRestartKubelet

// TryRestartKubelet attempts to restart the kubelet service
func TryRestartKubelet() {
	// If we notice that the kubelet service is inactive, try to start it
	initSystem, err := initsystem.GetInitSystem()
	if err != nil {
		fmt.Println("[kubelet-start] No supported init system detected, won't make sure the kubelet not running for a short period of time while setting up configuration for it.")
		return
	}

	if !initSystem.ServiceExists(kubeadmconstants.Kubelet) {
		fmt.Println("[kubelet-start] Couldn't detect a kubelet service, can't make sure the kubelet not running for a short period of time while setting up configuration for it.")
	}

	// This runs "systemctl daemon-reload && systemctl stop kubelet"
	if err := initSystem.ServiceRestart(kubeadmconstants.Kubelet); err != nil {
		klog.Warningf("[kubelet-start] WARNING: unable to restart the kubelet service momentarily: [%v]\n", err)
	}
}


MarkControlPlane

// MarkControlPlane taints the control-plane and sets the control-plane label
func MarkControlPlane(client clientset.Interface, controlPlaneName string, taints []v1.Taint) error {
	fmt.Printf("[mark-control-plane] Marking the node %s as control-plane by adding the labels: %v\n",
		controlPlaneName, labelsToAdd)

	if len(taints) > 0 {
		taintStrs := []string{}
		for _, taint := range taints {
			taintStrs = append(taintStrs, taint.ToString())
		}
		fmt.Printf("[mark-control-plane] Marking the node %s as control-plane by adding the taints %v\n", controlPlaneName, taintStrs)
	}

	return apiclient.PatchNode(client, controlPlaneName, func(n *v1.Node) {
		markControlPlaneNode(n, taints)
	})
}


taintExists

func taintExists(taint v1.Taint, taints []v1.Taint) bool {
	for _, t := range taints {
		if t == taint {
			return true
		}
	}

	return false
}


markControlPlaneNode

func markControlPlaneNode(n *v1.Node, taints []v1.Taint) {
	for _, label := range labelsToAdd {
		n.ObjectMeta.Labels[label] = ""
	}

	for _, nt := range n.Spec.Taints {
		if !taintExists(nt, taints) {
			taints = append(taints, nt)
		}
	}

	n.Spec.Taints = taints
}


AnnotateCRISocket

// AnnotateCRISocket annotates the node with the given crisocket
func AnnotateCRISocket(client clientset.Interface, nodeName string, criSocket string) error {

	klog.V(1).Infof("[patchnode] Uploading the CRI Socket information %q to the Node API object %q as an annotation\n", criSocket, nodeName)

	return apiclient.PatchNode(client, nodeName, func(n *v1.Node) {
		annotateNodeWithCRISocket(n, criSocket)
	})
}


annotateNodeWithCRISocket

func annotateNodeWithCRISocket(n *v1.Node, criSocket string) {
	if n.ObjectMeta.Annotations == nil {
		n.ObjectMeta.Annotations = make(map[string]string)
	}
	n.ObjectMeta.Annotations[constants.AnnotationKubeadmCRISocket] = criSocket
}


CanUpgradeKubelets

// CanUpgradeKubelets returns whether an upgrade of any kubelet in the cluster is possible
func (u *Upgrade) CanUpgradeKubelets() bool {
	// If there are multiple different versions now, an upgrade is possible (even if only for a subset of the nodes)
	if len(u.Before.KubeletVersions) > 1 {
		return true
	}
	// Don't report something available for upgrade if we don't know the current state
	if len(u.Before.KubeletVersions) == 0 {
		return false
	}

	// if the same version number existed both before and after, we don't have to upgrade it
	_, sameVersionFound := u.Before.KubeletVersions[u.After.KubeVersion]
	return !sameVersionFound
}


CanUpgradeEtcd

// CanUpgradeEtcd returns whether an upgrade of etcd is possible
func (u *Upgrade) CanUpgradeEtcd() bool {
	return u.Before.EtcdVersion != u.After.EtcdVersion
}


GetAvailableUpgrades

// GetAvailableUpgrades fetches all versions from the specified VersionGetter and computes which
// kinds of upgrades can be performed
func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesAllowed, rcUpgradesAllowed, externalEtcd bool, client clientset.Interface, manifestsDir string, printer output.Printer) ([]Upgrade, error) {
	printer.Printf("[upgrade] Fetching available versions to upgrade to\n")

	// Collect the upgrades kubeadm can do in this list
	var upgrades []Upgrade

	// Get the cluster version
	clusterVersionStr, clusterVersion, err := versionGetterImpl.ClusterVersion()
	if err != nil {
		return upgrades, err
	}
	printer.Printf("[upgrade/versions] Cluster version: %s\n", clusterVersionStr)

	// Get current kubeadm CLI version
	kubeadmVersionStr, kubeadmVersion, err := versionGetterImpl.KubeadmVersion()
	if err != nil {
		return upgrades, err
	}
	printer.Printf("[upgrade/versions] kubeadm version: %s\n", kubeadmVersionStr)

	// Get and output the current latest stable version
	stableVersionStr, stableVersion, err := versionGetterImpl.VersionFromCILabel("stable", "stable version")
	if err != nil {
		klog.Warningf("[upgrade/versions] WARNING: %v\n", err)
		klog.Warningf("[upgrade/versions] WARNING: Falling back to current kubeadm version as latest stable version")
		stableVersionStr, stableVersion = kubeadmVersionStr, kubeadmVersion
	} else {
		printer.Printf("[upgrade/versions] Target version: %s\n", stableVersionStr)
	}

	// Get the kubelet versions in the cluster
	kubeletVersions, err := versionGetterImpl.KubeletVersions()
	if err != nil {
		return upgrades, err
	}

	// Get current stacked etcd version on the local node
	var etcdVersion string
	if !externalEtcd {
		etcdVersion, err = GetEtcdImageTagFromStaticPod(manifestsDir)
		if err != nil {
			return upgrades, err
		}
	}

	dnsVersion, err := dns.DeployedDNSAddon(client)
	if err != nil {
		return nil, err
	}

	// Construct a descriptor for the current state of the world
	beforeState := ClusterState{
		KubeVersion:     clusterVersionStr,
		DNSVersion:      dnsVersion,
		KubeadmVersion:  kubeadmVersionStr,
		KubeletVersions: kubeletVersions,
		EtcdVersion:     etcdVersion,
	}

	// Do a "dumb guess" that a new minor upgrade is available just because the latest stable version is higher than the cluster version
	// This guess will be corrected once we know if there is a patch version available
	canDoMinorUpgrade := clusterVersion.LessThan(stableVersion)

	// A patch version doesn't exist if the cluster version is higher than or equal to the current stable version
	// in the case that a user is trying to upgrade from, let's say, v1.8.0-beta.2 to v1.8.0-rc.1 (given we support such upgrades experimentally)
	// a stable-1.8 branch doesn't exist yet. Hence this check.
	if patchVersionBranchExists(clusterVersion, stableVersion) {
		currentBranch := getBranchFromVersion(clusterVersionStr)
		versionLabel := fmt.Sprintf("stable-%s", currentBranch)
		description := fmt.Sprintf("version in the v%s series", currentBranch)

		// Get and output the latest patch version for the cluster branch
		patchVersionStr, patchVersion, err := versionGetterImpl.VersionFromCILabel(versionLabel, description)
		if err != nil {
			klog.Warningf("[upgrade/versions] WARNING: %v\n", err)
		} else {
			printer.Printf("[upgrade/versions] Latest %s: %s\n", description, patchVersionStr)

			// Check if a minor version upgrade is possible when a patch release exists
			// It's only possible if the latest patch version is higher than the current patch version
			// If that's the case, they must be on different branches => a newer minor version can be upgraded to
			canDoMinorUpgrade = minorUpgradePossibleWithPatchRelease(stableVersion, patchVersion)

			// If the cluster version is lower than the newest patch version, we should inform about the possible upgrade
			if patchUpgradePossible(clusterVersion, patchVersion) {

				// The kubeadm version has to be upgraded to the latest patch version
				newKubeadmVer := patchVersionStr
				if kubeadmVersion.AtLeast(patchVersion) {
					// In this case, the kubeadm CLI version is new enough. Don't display an update suggestion for kubeadm by making .NewKubeadmVersion equal .CurrentKubeadmVersion
					newKubeadmVer = kubeadmVersionStr
				}

				upgrades = append(upgrades, Upgrade{
					Description: description,
					Before:      beforeState,
					After: ClusterState{
						KubeVersion:    patchVersionStr,
						DNSVersion:     kubeadmconstants.CoreDNSVersion,
						KubeadmVersion: newKubeadmVer,
						EtcdVersion:    getSuggestedEtcdVersion(externalEtcd, patchVersionStr),
						// KubeletVersions is unset here as it is not used anywhere in .After
					},
				})
			}
		}
	}

	if canDoMinorUpgrade {
		upgrades = append(upgrades, Upgrade{
			Description: "stable version",
			Before:      beforeState,
			After: ClusterState{
				KubeVersion:    stableVersionStr,
				DNSVersion:     kubeadmconstants.CoreDNSVersion,
				KubeadmVersion: stableVersionStr,
				EtcdVersion:    getSuggestedEtcdVersion(externalEtcd, stableVersionStr),
				// KubeletVersions is unset here as it is not used anywhere in .After
			},
		})
	}

	if experimentalUpgradesAllowed || rcUpgradesAllowed {
		// dl.k8s.io/release/latest.txt is ALWAYS an alpha.X version
		// dl.k8s.io/release/latest-1.X.txt is first v1.X.0-alpha.0 -> v1.X.0-alpha.Y, then v1.X.0-beta.0 to v1.X.0-beta.Z, then v1.X.0-rc.1 to v1.X.0-rc.W.
		// After the v1.X.0 release, latest-1.X.txt is always a beta.0 version. Let's say the latest stable version on the v1.7 branch is v1.7.3, then the
		// latest-1.7 version is v1.7.4-beta.0

		// Worth noticing is that when the release-1.X branch is cut; there are two versions tagged: v1.X.0-beta.0 AND v1.(X+1).alpha.0
		// The v1.(X+1).alpha.0 is pretty much useless and should just be ignored, as more betas may be released that have more features than the initial v1.(X+1).alpha.0

		// So what we do below is getting the latest overall version, always an v1.X.0-alpha.Y version. Then we get latest-1.(X-1) version. This version may be anything
		// between v1.(X-1).0-beta.0 and v1.(X-1).Z-beta.0. At some point in time, latest-1.(X-1) will point to v1.(X-1).0-rc.1. Then we should show it.

		// The flow looks like this (with time on the X axis):
		// v1.8.0-alpha.1 -> v1.8.0-alpha.2 -> v1.8.0-alpha.3 | release-1.8 branch | v1.8.0-beta.0 -> v1.8.0-beta.1 -> v1.8.0-beta.2 -> v1.8.0-rc.1 -> v1.8.0 -> v1.8.1
		//                                                                           v1.9.0-alpha.0                                             -> v1.9.0-alpha.1 -> v1.9.0-alpha.2

		// Get and output the current latest unstable version
		latestVersionStr, latestVersion, err := versionGetterImpl.VersionFromCILabel("latest", "experimental version")
		if err != nil {
			return upgrades, err
		}
		fmt.Printf("[upgrade/versions] Latest %s: %s\n", "experimental version", latestVersionStr)

		minorUnstable := latestVersion.Components()[1]
		// Get and output the current latest unstable version
		previousBranch := fmt.Sprintf("latest-1.%d", minorUnstable-1)
		previousBranchLatestVersionStr, previousBranchLatestVersion, err := versionGetterImpl.VersionFromCILabel(previousBranch, "previous version")
		if err != nil {
			return upgrades, err
		}
		fmt.Printf("[upgrade/versions] Latest %s: %s\n", "previous version", previousBranchLatestVersionStr)

		// If that previous latest version is an RC, RCs are allowed and the cluster version is lower than the RC version, show the upgrade
		if rcUpgradesAllowed && rcUpgradePossible(clusterVersion, previousBranchLatestVersion) {
			upgrades = append(upgrades, Upgrade{
				Description: "release candidate version",
				Before:      beforeState,
				After: ClusterState{
					KubeVersion:    previousBranchLatestVersionStr,
					DNSVersion:     kubeadmconstants.CoreDNSVersion,
					KubeadmVersion: previousBranchLatestVersionStr,
					EtcdVersion:    getSuggestedEtcdVersion(externalEtcd, previousBranchLatestVersionStr),
					// KubeletVersions is unset here as it is not used anywhere in .After
				},
			})
		}

		// Show the possibility if experimental upgrades are allowed
		if experimentalUpgradesAllowed && clusterVersion.LessThan(latestVersion) {

			// Default to assume that the experimental version to show is the unstable one
			unstableKubeVersion := latestVersionStr

			// Ẃe should not display alpha.0. The previous branch's beta/rc versions are more relevant due how the kube branching process works.
			if latestVersion.PreRelease() == "alpha.0" {
				unstableKubeVersion = previousBranchLatestVersionStr
			}

			upgrades = append(upgrades, Upgrade{
				Description: "experimental version",
				Before:      beforeState,
				After: ClusterState{
					KubeVersion:    unstableKubeVersion,
					DNSVersion:     kubeadmconstants.CoreDNSVersion,
					KubeadmVersion: unstableKubeVersion,
					EtcdVersion:    getSuggestedEtcdVersion(externalEtcd, unstableKubeVersion),
					// KubeletVersions is unset here as it is not used anywhere in .After
				},
			})
		}
	}

	// Add a newline in the end of this output to leave some space to the next output section
	printer.Println()

	return upgrades, nil
}


getBranchFromVersion

func getBranchFromVersion(version string) string {
	v := versionutil.MustParseGeneric(version)
	return fmt.Sprintf("%d.%d", v.Major(), v.Minor())
}


patchVersionBranchExists

func patchVersionBranchExists(clusterVersion, stableVersion *versionutil.Version) bool {
	return stableVersion.AtLeast(clusterVersion)
}


patchUpgradePossible

func patchUpgradePossible(clusterVersion, patchVersion *versionutil.Version) bool {
	return clusterVersion.LessThan(patchVersion)
}


rcUpgradePossible

func rcUpgradePossible(clusterVersion, previousBranchLatestVersion *versionutil.Version) bool {
	return strings.HasPrefix(previousBranchLatestVersion.PreRelease(), "rc") && clusterVersion.LessThan(previousBranchLatestVersion)
}


minorUpgradePossibleWithPatchRelease

func minorUpgradePossibleWithPatchRelease(stableVersion, patchVersion *versionutil.Version) bool {
	return patchVersion.LessThan(stableVersion)
}


getSuggestedEtcdVersion

func getSuggestedEtcdVersion(externalEtcd bool, kubernetesVersion string) string {
	if externalEtcd {
		return ""
	}
	etcdVersion, warning, err := kubeadmconstants.EtcdSupportedVersion(kubeadmconstants.SupportedEtcdVersion, kubernetesVersion)
	if err != nil {
		klog.Warningf("[upgrade/versions] could not retrieve an etcd version for the target Kubernetes version: %v", err)
		return "N/A"
	}
	if warning != nil {
		klog.Warningf("[upgrade/versions] %v", warning)
	}
	return etcdVersion.String()
}


Check

// Check is part of the preflight.Checker interface
func (c *healthCheck) Check() (warnings, errors []error) {
	if err := c.f(c.client, c.cfg); err != nil {
		return nil, []error{err}
	}
	return nil, nil
}


Name

// Name is part of the preflight.Checker interface
func (c *healthCheck) Name() string {
	return c.name
}


CheckClusterHealth

// CheckClusterHealth makes sure:
// - the API /healthz endpoint is healthy
// - all control-plane Nodes are Ready
// - (if static pod-hosted) that all required Static Pod manifests exist on disk
func CheckClusterHealth(client clientset.Interface, cfg *kubeadmapi.ClusterConfiguration, ignoreChecksErrors sets.Set[string]) error {
	fmt.Println("[upgrade] Running cluster health checks")

	healthChecks := []preflight.Checker{
		&healthCheck{
			name:   "CreateJob",
			client: client,
			cfg:    cfg,
			f:      createJob,
		},
		&healthCheck{
			name:   "ControlPlaneNodesReady",
			client: client,
			f:      controlPlaneNodesReady,
		},
		&healthCheck{
			name:   "StaticPodManifest",
			client: client,
			cfg:    cfg,
			f:      staticPodManifestHealth,
		},
	}

	return preflight.RunChecks(healthChecks, os.Stderr, ignoreChecksErrors)
}


createJob

// CreateJob is a check that verifies that a Job can be created in the cluster
func createJob(client clientset.Interface, cfg *kubeadmapi.ClusterConfiguration) (lastError error) {
	const (
		jobName = "upgrade-health-check"
		ns      = metav1.NamespaceSystem
		timeout = 15 * time.Second
	)

	// If client.Discovery().RESTClient() is nil, the fake client is used.
	// Return early because the kubeadm dryrun dynamic client only handles the core/v1 GroupVersion.
	if client.Discovery().RESTClient() == nil {
		fmt.Printf("[upgrade/health] Would create the Job %q in namespace %q and wait until it completes\n", jobName, ns)
		return nil
	}

	// Prepare Job
	job := &batchv1.Job{
		ObjectMeta: metav1.ObjectMeta{
			Name:      jobName,
			Namespace: ns,
		},
		Spec: batchv1.JobSpec{
			BackoffLimit: pointer.Int32(0),
			Template: v1.PodTemplateSpec{
				Spec: v1.PodSpec{
					RestartPolicy: v1.RestartPolicyNever,
					SecurityContext: &v1.PodSecurityContext{
						RunAsUser:    pointer.Int64(999),
						RunAsGroup:   pointer.Int64(999),
						RunAsNonRoot: pointer.Bool(true),
					},
					Tolerations: []v1.Toleration{
						{
							Key:    constants.LabelNodeRoleControlPlane,
							Effect: v1.TaintEffectNoSchedule,
						},
					},
					Containers: []v1.Container{
						{
							Name:  jobName,
							Image: images.GetPauseImage(cfg),
							Args:  []string{"-v"},
						},
					},
				},
			},
		},
	}

	// Check if the Job already exists and delete it
	if _, err := client.BatchV1().Jobs(ns).Get(context.TODO(), jobName, metav1.GetOptions{}); err == nil {
		if err = deleteHealthCheckJob(client, ns, jobName); err != nil {
			return err
		}
	}

	// Cleanup the Job on exit
	defer func() {
		lastError = deleteHealthCheckJob(client, ns, jobName)
	}()

	// Create the Job, but retry in case it is being currently deleted
	klog.V(2).Infof("Creating Job %q in the namespace %q", jobName, ns)
	err := wait.PollImmediate(time.Second*1, timeout, func() (bool, error) {
		if _, err := client.BatchV1().Jobs(ns).Create(context.TODO(), job, metav1.CreateOptions{}); err != nil {
			klog.V(2).Infof("Could not create Job %q in the namespace %q, retrying: %v", jobName, ns, err)
			lastError = err
			return false, nil
		}
		return true, nil
	})
	if err != nil {
		return errors.Wrapf(lastError, "could not create Job %q in the namespace %q", jobName, ns)
	}

	// Waiting and manually deleting the Job is a workaround to not enabling the TTL controller.
	// TODO: refactor this if the TTL controller is enabled in kubeadm once it goes Beta.

	// Wait for the Job to complete
	err = wait.PollImmediate(time.Second*1, timeout, func() (bool, error) {
		job, err := client.BatchV1().Jobs(ns).Get(context.TODO(), jobName, metav1.GetOptions{})
		if err != nil {
			lastError = err
			klog.V(2).Infof("could not get Job %q in the namespace %q, retrying: %v", jobName, ns, err)
			return false, nil
		}
		for _, cond := range job.Status.Conditions {
			if cond.Type == batchv1.JobComplete {
				return true, nil
			}
		}
		lastError = errors.Errorf("no condition of type %v", batchv1.JobComplete)
		klog.V(2).Infof("Job %q in the namespace %q is not yet complete, retrying", jobName, ns)
		return false, nil
	})
	if err != nil {
		return errors.Wrapf(lastError, "Job %q in the namespace %q did not complete in %v", jobName, ns, timeout)
	}

	klog.V(2).Infof("Job %q in the namespace %q completed", jobName, ns)

	return nil
}


deleteHealthCheckJob

func deleteHealthCheckJob(client clientset.Interface, ns, jobName string) error {
	klog.V(2).Infof("Deleting Job %q in the namespace %q", jobName, ns)
	propagation := metav1.DeletePropagationForeground
	if err := client.BatchV1().Jobs(ns).Delete(context.TODO(), jobName, metav1.DeleteOptions{PropagationPolicy: &propagation}); err != nil {
		return errors.Wrapf(err, "could not delete Job %q in the namespace %q", jobName, ns)
	}
	return nil
}


controlPlaneNodesReady

// controlPlaneNodesReady checks whether all control-plane Nodes in the cluster are in the Running state
func controlPlaneNodesReady(client clientset.Interface, _ *kubeadmapi.ClusterConfiguration) error {
	selectorControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{
		constants.LabelNodeRoleControlPlane: "",
	}))
	nodes, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{
		LabelSelector: selectorControlPlane.String(),
	})
	if err != nil {
		return errors.Wrapf(err, "could not list nodes labeled with %q", constants.LabelNodeRoleControlPlane)
	}

	notReadyControlPlanes := getNotReadyNodes(nodes.Items)
	if len(notReadyControlPlanes) != 0 {
		return errors.Errorf("there are NotReady control-planes in the cluster: %v", notReadyControlPlanes)
	}
	return nil
}


staticPodManifestHealth

// staticPodManifestHealth makes sure the required static pods are presents
func staticPodManifestHealth(_ clientset.Interface, _ *kubeadmapi.ClusterConfiguration) error {
	var nonExistentManifests []string
	for _, component := range constants.ControlPlaneComponents {
		manifestFile := constants.GetStaticPodFilepath(component, constants.GetStaticPodDirectory())
		if _, err := os.Stat(manifestFile); os.IsNotExist(err) {
			nonExistentManifests = append(nonExistentManifests, manifestFile)
		}
	}
	if len(nonExistentManifests) == 0 {
		return nil
	}
	return errors.Errorf("The control plane seems to be Static Pod-hosted, but some of the manifests don't seem to exist on disk. This probably means you're running 'kubeadm upgrade' on a remote machine, which is not supported for a Static Pod-hosted cluster. Manifest files not found: %v", nonExistentManifests)
}


getNotReadyNodes

// getNotReadyNodes returns a string slice of nodes in the cluster that are NotReady
func getNotReadyNodes(nodes []v1.Node) []string {
	var notReadyNodes []string
	for _, node := range nodes {
		for _, condition := range node.Status.Conditions {
			if condition.Type == v1.NodeReady && condition.Status != v1.ConditionTrue {
				notReadyNodes = append(notReadyNodes, node.ObjectMeta.Name)
			}
		}
	}
	return notReadyNodes
}


EnforceVersionPolicies

// EnforceVersionPolicies enforces that the proposed new version is compatible with all the different version skew policies
func EnforceVersionPolicies(versionGetter VersionGetter, newK8sVersionStr string, newK8sVersion *version.Version, allowExperimentalUpgrades, allowRCUpgrades bool) *VersionSkewPolicyErrors {

	skewErrors := &VersionSkewPolicyErrors{
		Mandatory: []error{},
		Skippable: []error{},
	}

	clusterVersionStr, clusterVersion, err := versionGetter.ClusterVersion()
	if err != nil {
		// This case can't be forced: kubeadm has to be able to lookup cluster version for upgrades to work
		skewErrors.Mandatory = append(skewErrors.Mandatory, errors.Wrap(err, "Unable to fetch cluster version"))
		return skewErrors
	}
	fmt.Printf("[upgrade/versions] Cluster version: %s\n", clusterVersionStr)

	kubeadmVersionStr, kubeadmVersion, err := versionGetter.KubeadmVersion()
	if err != nil {
		// This case can't be forced: kubeadm has to be able to lookup its version for upgrades to work
		skewErrors.Mandatory = append(skewErrors.Mandatory, errors.Wrap(err, "Unable to fetch kubeadm version"))
		return skewErrors
	}
	fmt.Printf("[upgrade/versions] kubeadm version: %s\n", kubeadmVersionStr)

	kubeletVersions, err := versionGetter.KubeletVersions()
	if err != nil {
		// This is a non-critical error; continue although kubeadm couldn't look this up
		skewErrors.Skippable = append(skewErrors.Skippable, errors.Wrap(err, "Unable to fetch kubelet version"))
	}

	// Make sure the new version is a supported version (higher than the minimum one supported)
	if !newK8sVersion.AtLeast(constants.MinimumControlPlaneVersion) {
		// This must not happen, kubeadm always supports a minimum version; and we can't go below that
		skewErrors.Mandatory = append(skewErrors.Mandatory, errors.Errorf("Specified version to upgrade to %q is equal to or lower than the minimum supported version %q. Please specify a higher version to upgrade to", newK8sVersionStr, clusterVersionStr))
	}

	// kubeadm doesn't support upgrades between two minor versions; e.g. a v1.7 -> v1.9 upgrade is not supported right away
	if newK8sVersion.Minor() > clusterVersion.Minor()+MaximumAllowedMinorVersionUpgradeSkew {
		tooLargeUpgradeSkewErr := errors.Errorf("Specified version to upgrade to %q is too high; kubeadm can upgrade only %d minor version at a time", newK8sVersionStr, MaximumAllowedMinorVersionUpgradeSkew)
		// If the version that we're about to upgrade to is a released version, we should fully enforce this policy
		// If the version is a CI/dev/experimental version, it's okay to jump two minor version steps, but then require the -f flag
		if len(newK8sVersion.PreRelease()) == 0 {
			skewErrors.Mandatory = append(skewErrors.Mandatory, tooLargeUpgradeSkewErr)
		} else {
			skewErrors.Skippable = append(skewErrors.Skippable, tooLargeUpgradeSkewErr)
		}
	}

	// kubeadm doesn't support downgrades between two minor versions; e.g. a v1.9 -> v1.7 downgrade is not supported right away
	if newK8sVersion.Minor() < clusterVersion.Minor()-MaximumAllowedMinorVersionDowngradeSkew {
		tooLargeDowngradeSkewErr := errors.Errorf("Specified version to downgrade to %q is too low; kubeadm can downgrade only %d minor version at a time", newK8sVersionStr, MaximumAllowedMinorVersionDowngradeSkew)
		// If the version that we're about to downgrade to is a released version, we should fully enforce this policy
		// If the version is a CI/dev/experimental version, it's okay to jump two minor version steps, but then require the -f flag
		if len(newK8sVersion.PreRelease()) == 0 {
			skewErrors.Mandatory = append(skewErrors.Mandatory, tooLargeDowngradeSkewErr)
		} else {
			skewErrors.Skippable = append(skewErrors.Skippable, tooLargeDowngradeSkewErr)
		}
	}

	// If the kubeadm version is lower than what we want to upgrade to; error
	if kubeadmVersion.LessThan(newK8sVersion) {
		if newK8sVersion.Minor() > kubeadmVersion.Minor() {
			tooLargeKubeadmSkew := errors.Errorf("Specified version to upgrade to %q is at least one minor release higher than the kubeadm minor release (%d > %d). Such an upgrade is not supported", newK8sVersionStr, newK8sVersion.Minor(), kubeadmVersion.Minor())
			// This is unsupported; kubeadm has no idea how it should handle a newer minor release than itself
			// If the version is a CI/dev/experimental version though, lower the severity of this check, but then require the -f flag
			if len(newK8sVersion.PreRelease()) == 0 {
				skewErrors.Mandatory = append(skewErrors.Mandatory, tooLargeKubeadmSkew)
			} else {
				skewErrors.Skippable = append(skewErrors.Skippable, tooLargeKubeadmSkew)
			}
		} else {
			// Upgrading to a higher patch version than kubeadm is ok if the user specifies --force. Not recommended, but possible.
			skewErrors.Skippable = append(skewErrors.Skippable, errors.Errorf("Specified version to upgrade to %q is higher than the kubeadm version %q. Upgrade kubeadm first using the tool you used to install kubeadm", newK8sVersionStr, kubeadmVersionStr))
		}
	}

	if kubeadmVersion.Major() > newK8sVersion.Major() ||
		kubeadmVersion.Minor() > newK8sVersion.Minor() {
		skewErrors.Skippable = append(skewErrors.Skippable, errors.Errorf("Kubeadm version %s can only be used to upgrade to Kubernetes version %d.%d", kubeadmVersionStr, kubeadmVersion.Major(), kubeadmVersion.Minor()))
	}

	// Detect if the version is unstable and the user didn't allow that
	if err = detectUnstableVersionError(newK8sVersion, newK8sVersionStr, allowExperimentalUpgrades, allowRCUpgrades); err != nil {
		skewErrors.Skippable = append(skewErrors.Skippable, err)
	}

	// Detect if there are too old kubelets in the cluster
	// Check for nil here since this is the only case where kubeletVersions can be nil; if KubeletVersions() returned an error
	// However, it's okay to skip that check
	if kubeletVersions != nil {
		if err = detectTooOldKubelets(newK8sVersion, kubeletVersions); err != nil {
			skewErrors.Skippable = append(skewErrors.Skippable, err)
		}
	}

	// If we did not see any errors, return nil
	if len(skewErrors.Skippable) == 0 && len(skewErrors.Mandatory) == 0 {
		return nil
	}

	// Uh oh, we encountered one or more errors, return them
	return skewErrors
}


detectUnstableVersionError

// detectUnstableVersionError is a helper function for detecting if the unstable version (if specified) is allowed to be used
func detectUnstableVersionError(newK8sVersion *version.Version, newK8sVersionStr string, allowExperimentalUpgrades, allowRCUpgrades bool) error {
	// Short-circuit quickly if this is not an unstable version
	if len(newK8sVersion.PreRelease()) == 0 {
		return nil
	}
	// If the user has specified that unstable versions are fine, then no error should be returned
	if allowExperimentalUpgrades {
		return nil
	}
	// If this is a release candidate and we allow such ones, everything's fine
	if strings.HasPrefix(newK8sVersion.PreRelease(), "rc") && allowRCUpgrades {
		return nil
	}

	return errors.Errorf("Specified version to upgrade to %q is an unstable version and such upgrades weren't allowed via setting the --allow-*-upgrades flags", newK8sVersionStr)
}


detectTooOldKubelets

// detectTooOldKubelets errors out if the kubelet versions are so old that an unsupported skew would happen if the cluster was upgraded
func detectTooOldKubelets(newK8sVersion *version.Version, kubeletVersions map[string]uint16) error {
	var tooOldKubeletVersions []string
	for versionStr := range kubeletVersions {

		kubeletVersion, err := version.ParseSemantic(versionStr)
		if err != nil {
			return errors.Errorf("couldn't parse kubelet version %s", versionStr)
		}

		if newK8sVersion.Minor() > kubeletVersion.Minor()+MaximumAllowedMinorVersionKubeletSkew {
			tooOldKubeletVersions = append(tooOldKubeletVersions, versionStr)
		}
	}
	if len(tooOldKubeletVersions) == 0 {
		return nil
	}

	return errors.Errorf("There are kubelets in this cluster that are too old that have these versions %v", tooOldKubeletVersions)
}


PerformPostUpgradeTasks

// PerformPostUpgradeTasks runs nearly the same functions as 'kubeadm init' would do
// Note that the mark-control-plane phase is left out, not needed, and no token is created as that doesn't belong to the upgrade
func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, patchesDir string, dryRun bool, out io.Writer) error {
	var errs []error

	// Upload currently used configuration to the cluster
	// Note: This is done right in the beginning of cluster initialization; as we might want to make other phases
	// depend on centralized information from this source in the future
	if err := uploadconfig.UploadConfiguration(cfg, client); err != nil {
		errs = append(errs, err)
	}

	// Create the new, version-branched kubelet ComponentConfig ConfigMap
	if err := kubeletphase.CreateConfigMap(&cfg.ClusterConfiguration, patchesDir, client); err != nil {
		errs = append(errs, errors.Wrap(err, "error creating kubelet configuration ConfigMap"))
	}

	// Write the new kubelet config down to disk and the env file if needed
	if err := WriteKubeletConfigFiles(cfg, patchesDir, dryRun, out); err != nil {
		errs = append(errs, err)
	}

	// Annotate the node with the crisocket information, sourced either from the InitConfiguration struct or
	// --cri-socket.
	// TODO: In the future we want to use something more official like NodeStatus or similar for detecting this properly
	if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil {
		errs = append(errs, errors.Wrap(err, "error uploading crisocket"))
	}

	// Create RBAC rules that makes the bootstrap tokens able to get nodes
	if err := nodebootstraptoken.AllowBoostrapTokensToGetNodes(client); err != nil {
		errs = append(errs, err)
	}

	// Create/update RBAC rules that makes the bootstrap tokens able to post CSRs
	if err := nodebootstraptoken.AllowBootstrapTokensToPostCSRs(client); err != nil {
		errs = append(errs, err)
	}

	// Create/update RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically
	if err := nodebootstraptoken.AutoApproveNodeBootstrapTokens(client); err != nil {
		errs = append(errs, err)
	}

	// Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically
	if err := nodebootstraptoken.AutoApproveNodeCertificateRotation(client); err != nil {
		errs = append(errs, err)
	}

	// TODO: Is this needed to do here? I think that updating cluster info should probably be separate from a normal upgrade
	// Create the cluster-info ConfigMap with the associated RBAC rules
	// if err := clusterinfo.CreateBootstrapConfigMapIfNotExists(client, kubeadmconstants.GetAdminKubeConfigPath()); err != nil {
	// 	return err
	//}
	// Create/update RBAC rules that makes the cluster-info ConfigMap reachable
	if err := clusterinfo.CreateClusterInfoRBACRules(client); err != nil {
		errs = append(errs, err)
	}

	// If the coredns ConfigMap is missing, show a warning and assume that the
	// DNS addon was skipped during "kubeadm init", and that its redeployment on upgrade is not desired.
	//
	// TODO: remove this once "kubeadm upgrade apply" phases are supported:
	//   https://github.com/kubernetes/kubeadm/issues/1318
	var missingCoreDNSConfigMap bool
	if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(
		context.TODO(),
		kubeadmconstants.CoreDNSConfigMap,
		metav1.GetOptions{},
	); err != nil && apierrors.IsNotFound(err) {
		missingCoreDNSConfigMap = true
	}
	if missingCoreDNSConfigMap {
		klog.Warningf("the ConfigMaps %q in the namespace %q were not found. "+
			"Assuming that a DNS server was not deployed for this cluster. "+
			"Note that once 'kubeadm upgrade apply' supports phases you "+
			"will have to skip the DNS upgrade manually",
			kubeadmconstants.CoreDNSConfigMap,
			metav1.NamespaceSystem)
	} else {
		// Upgrade CoreDNS
		if err := dns.EnsureDNSAddon(&cfg.ClusterConfiguration, client, out, false); err != nil {
			errs = append(errs, err)
		}
	}

	// If the kube-proxy ConfigMap is missing, show a warning and assume that kube-proxy
	// was skipped during "kubeadm init", and that its redeployment on upgrade is not desired.
	//
	// TODO: remove this once "kubeadm upgrade apply" phases are supported:
	//   https://github.com/kubernetes/kubeadm/issues/1318
	if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(
		context.TODO(),
		kubeadmconstants.KubeProxyConfigMap,
		metav1.GetOptions{},
	); err != nil && apierrors.IsNotFound(err) {
		klog.Warningf("the ConfigMap %q in the namespace %q was not found. "+
			"Assuming that kube-proxy was not deployed for this cluster. "+
			"Note that once 'kubeadm upgrade apply' supports phases you "+
			"will have to skip the kube-proxy upgrade manually",
			kubeadmconstants.KubeProxyConfigMap,
			metav1.NamespaceSystem)
	} else {
		// Upgrade kube-proxy
		if err := proxy.EnsureProxyAddon(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, client, out, false); err != nil {
			errs = append(errs, err)
		}
	}

	return errorsutil.NewAggregate(errs)
}


WriteKubeletConfigFiles

func WriteKubeletConfigFiles(cfg *kubeadmapi.InitConfiguration, patchesDir string, dryRun bool, out io.Writer) error {
	// Set up the kubelet directory to use. If dry-running, this will return a fake directory
	kubeletDir, err := GetKubeletDir(dryRun)
	if err != nil {
		// The error here should never occur in reality, would only be thrown if /tmp doesn't exist on the machine.
		return err
	}

	// Create a copy of the kubelet config file in the /etc/kubernetes/tmp/ folder.
	backupDir, err := kubeadmconstants.CreateTempDirForKubeadm(kubeadmconstants.KubernetesDir, "kubeadm-kubelet-config")
	if err != nil {
		return err
	}
	src := filepath.Join(kubeletDir, kubeadmconstants.KubeletConfigurationFileName)
	dest := filepath.Join(backupDir, kubeadmconstants.KubeletConfigurationFileName)

	if !dryRun {
		fmt.Printf("[upgrade] Backing up kubelet config file to %s\n", dest)
		err := kubeadmutil.CopyFile(src, dest)
		if err != nil {
			return errors.Wrap(err, "error backing up the kubelet config file")
		}
	} else {
		fmt.Printf("[dryrun] Would back up kubelet config file to %s\n", dest)
	}

	errs := []error{}
	// Write the configuration for the kubelet down to disk so the upgraded kubelet can start with fresh config
	if err := kubeletphase.WriteConfigToDisk(&cfg.ClusterConfiguration, kubeletDir, patchesDir, out); err != nil {
		errs = append(errs, errors.Wrap(err, "error writing kubelet configuration to file"))
	}

	if dryRun { // Print what contents would be written
		err := dryrunutil.PrintDryRunFile(kubeadmconstants.KubeletConfigurationFileName, kubeletDir, kubeadmconstants.KubeletRunDirectory, os.Stdout)
		if err != nil {
			errs = append(errs, errors.Wrap(err, "error printing files on dryrun"))
		}
	}
	return errorsutil.NewAggregate(errs)
}


GetKubeletDir

// GetKubeletDir gets the kubelet directory based on whether the user is dry-running this command or not.
func GetKubeletDir(dryRun bool) (string, error) {
	if dryRun {
		return kubeadmconstants.CreateTempDirForKubeadm("", "kubeadm-upgrade-dryrun")
	}
	return kubeadmconstants.KubeletRunDirectory, nil
}


moveFiles

// moveFiles moves files from one directory to another.
func moveFiles(files map[string]string) error {
	filesToRecover := make(map[string]string, len(files))
	for from, to := range files {
		if err := os.Rename(from, to); err != nil {
			return rollbackFiles(filesToRecover, err)
		}
		filesToRecover[to] = from
	}
	return nil
}


rollbackFiles

// rollbackFiles moves the files back to the original directory.
func rollbackFiles(files map[string]string, originalErr error) error {
	errs := []error{originalErr}
	for from, to := range files {
		if err := os.Rename(from, to); err != nil {
			errs = append(errs, err)
		}
	}
	return errors.Errorf("couldn't move these files: %v. Got errors: %v", files, errorsutil.NewAggregate(errs))
}


Name

// Name is part of the preflight.Checker interface
func (c CoreDNSCheck) Name() string {
	return c.name
}


Check

// Check is part of the preflight.Checker interface
func (c CoreDNSCheck) Check() (warnings, errors []error) {
	if err := c.f(c.client); err != nil {
		return nil, []error{err}
	}
	return nil, nil
}


RunCoreDNSMigrationCheck

// RunCoreDNSMigrationCheck initializes checks related to CoreDNS migration.
func RunCoreDNSMigrationCheck(client clientset.Interface, ignorePreflightErrors sets.Set[string]) error {
	migrationChecks := []preflight.Checker{
		&CoreDNSCheck{
			name:   "CoreDNSUnsupportedPlugins",
			client: client,
			f:      checkUnsupportedPlugins,
		},
		&CoreDNSCheck{
			name:   "CoreDNSMigration",
			client: client,
			f:      checkMigration,
		},
	}

	return preflight.RunChecks(migrationChecks, os.Stderr, ignorePreflightErrors)
}


checkUnsupportedPlugins

// checkUnsupportedPlugins checks if there are any plugins included in the current configuration
// that are unsupported for migration.
func checkUnsupportedPlugins(client clientset.Interface) error {
	klog.V(1).Infoln("validating if there are any unsupported CoreDNS plugins in the Corefile")
	_, corefile, currentInstalledCoreDNSversion, err := dns.GetCoreDNSInfo(client)
	if err != nil {
		return err
	}

	currentInstalledCoreDNSversion = strings.TrimLeft(currentInstalledCoreDNSversion, "v")
	targetCoreDNSVersion := strings.TrimLeft(kubeadmconstants.CoreDNSVersion, "v")
	if currentInstalledCoreDNSversion == targetCoreDNSVersion {
		return nil
	}
	unsupportedCoreDNS, err := migration.Unsupported(currentInstalledCoreDNSversion, targetCoreDNSVersion, corefile)
	if err != nil {
		return err
	}
	if len(unsupportedCoreDNS) != 0 {
		var UnsupportedPlugins []string
		for _, unsup := range unsupportedCoreDNS {
			UnsupportedPlugins = append(UnsupportedPlugins, unsup.ToString())
		}
		fmt.Println("[preflight] The corefile contains plugins that kubeadm/CoreDNS does not know how to migrate. " +
			"Each plugin listed should be manually verified for compatibility with the newer version of CoreDNS. " +
			"Once ready, the upgrade can be initiated by skipping the preflight check. During the upgrade, " +
			"kubeadm will migrate the configuration while leaving the listed plugin configs untouched, " +
			"but cannot guarantee that they will work with the newer version of CoreDNS.")
		return errors.Errorf("CoreDNS cannot migrate the following plugins:\n%s", UnsupportedPlugins)
	}
	return nil
}


checkMigration

// checkMigration validates if migration of the current CoreDNS ConfigMap is possible.
func checkMigration(client clientset.Interface) error {
	klog.V(1).Infoln("validating if migration can be done for the current CoreDNS release.")
	_, corefile, currentInstalledCoreDNSversion, err := dns.GetCoreDNSInfo(client)
	if err != nil {
		return err
	}

	currentInstalledCoreDNSversion = strings.TrimLeft(currentInstalledCoreDNSversion, "v")
	_, err = migration.Migrate(currentInstalledCoreDNSversion, strings.TrimLeft(kubeadmconstants.CoreDNSVersion, "v"), corefile, false)
	return errors.Wrap(err, "CoreDNS will not be upgraded")
}


NewKubeStaticPodPathManager

// NewKubeStaticPodPathManager creates a new instance of KubeStaticPodPathManager
func NewKubeStaticPodPathManager(kubernetesDir, patchesDir, tempDir, backupDir, backupEtcdDir string, keepManifestDir, keepEtcdDir bool) StaticPodPathManager {
	return &KubeStaticPodPathManager{
		kubernetesDir:     kubernetesDir,
		patchesDir:        patchesDir,
		realManifestDir:   filepath.Join(kubernetesDir, constants.ManifestsSubDirName),
		tempManifestDir:   tempDir,
		backupManifestDir: backupDir,
		backupEtcdDir:     backupEtcdDir,
		keepManifestDir:   keepManifestDir,
		keepEtcdDir:       keepEtcdDir,
	}
}


// NewKubeStaticPodPathManagerUsingTempDirs creates a new instance of KubeStaticPodPathManager with temporary directories backing it
func NewKubeStaticPodPathManagerUsingTempDirs(kubernetesDir, patchesDir string, saveManifestsDir, saveEtcdDir bool) (StaticPodPathManager, error) {

	upgradedManifestsDir, err := constants.CreateTempDirForKubeadm(kubernetesDir, "kubeadm-upgraded-manifests")
	if err != nil {
		return nil, err
	}
	backupManifestsDir, err := constants.CreateTimestampDirForKubeadm(kubernetesDir, "kubeadm-backup-manifests")
	if err != nil {
		return nil, err
	}
	backupEtcdDir, err := constants.CreateTimestampDirForKubeadm(kubernetesDir, "kubeadm-backup-etcd")
	if err != nil {
		return nil, err
	}

	return NewKubeStaticPodPathManager(kubernetesDir, patchesDir, upgradedManifestsDir, backupManifestsDir, backupEtcdDir, saveManifestsDir, saveEtcdDir), nil
}


MoveFile

// MoveFile should move a file from oldPath to newPath
func (spm *KubeStaticPodPathManager) MoveFile(oldPath, newPath string) error {
	return os.Rename(oldPath, newPath)
}


KubernetesDir

// KubernetesDir should point to the directory Kubernetes owns for storing various configuration files
func (spm *KubeStaticPodPathManager) KubernetesDir() string {
	return spm.kubernetesDir
}


PatchesDir

// PatchesDir should point to the folder where patches for components are stored
func (spm *KubeStaticPodPathManager) PatchesDir() string {
	return spm.patchesDir
}


RealManifestPath

// RealManifestPath gets the file path for the component in the "real" static pod manifest directory used by the kubelet
func (spm *KubeStaticPodPathManager) RealManifestPath(component string) string {
	return constants.GetStaticPodFilepath(component, spm.realManifestDir)
}


RealManifestDir

// RealManifestDir should point to the static pod manifest directory used by the kubelet
func (spm *KubeStaticPodPathManager) RealManifestDir() string {
	return spm.realManifestDir
}


TempManifestPath

// TempManifestPath gets the file path for the component in the temporary directory created for generating new manifests for the upgrade
func (spm *KubeStaticPodPathManager) TempManifestPath(component string) string {
	return constants.GetStaticPodFilepath(component, spm.tempManifestDir)
}


TempManifestDir

// TempManifestDir should point to the temporary directory created for generating new manifests for the upgrade
func (spm *KubeStaticPodPathManager) TempManifestDir() string {
	return spm.tempManifestDir
}


BackupManifestPath

// BackupManifestPath gets the file path for the component in the backup directory used for backuping manifests during the transition
func (spm *KubeStaticPodPathManager) BackupManifestPath(component string) string {
	return constants.GetStaticPodFilepath(component, spm.backupManifestDir)
}


BackupManifestDir

// BackupManifestDir should point to the backup directory used for backuping manifests during the transition
func (spm *KubeStaticPodPathManager) BackupManifestDir() string {
	return spm.backupManifestDir
}


BackupEtcdDir

// BackupEtcdDir should point to the backup directory used for backuping manifests during the transition
func (spm *KubeStaticPodPathManager) BackupEtcdDir() string {
	return spm.backupEtcdDir
}


CleanupDirs

// CleanupDirs cleans up all temporary directories except those the user has requested to keep around
func (spm *KubeStaticPodPathManager) CleanupDirs() error {
	var errlist []error
	if err := os.RemoveAll(spm.TempManifestDir()); err != nil {
		errlist = append(errlist, err)
	}
	if !spm.keepManifestDir {
		if err := os.RemoveAll(spm.BackupManifestDir()); err != nil {
			errlist = append(errlist, err)
		}
	}

	if !spm.keepEtcdDir {
		if err := os.RemoveAll(spm.BackupEtcdDir()); err != nil {
			errlist = append(errlist, err)
		}
	}

	return utilerrors.NewAggregate(errlist)
}


upgradeComponent

func upgradeComponent(component string, certsRenewMgr *renewal.Manager, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, beforePodHash string, recoverManifests map[string]string) error {
	// Special treatment is required for etcd case, when rollbackOldManifests should roll back etcd
	// manifests only for the case when component is Etcd
	recoverEtcd := false
	if component == constants.Etcd {
		recoverEtcd = true
	}

	fmt.Printf("[upgrade/staticpods] Preparing for %q upgrade\n", component)

	// The old manifest is here; in the /etc/kubernetes/manifests/
	currentManifestPath := pathMgr.RealManifestPath(component)
	// The new, upgraded manifest will be written here
	newManifestPath := pathMgr.TempManifestPath(component)
	// The old manifest will be moved here; into a subfolder of the temporary directory
	// If a rollback is needed, these manifests will be put back to where they where initially
	backupManifestPath := pathMgr.BackupManifestPath(component)

	// Store the backup path in the recover list. If something goes wrong now, this component will be rolled back.
	recoverManifests[component] = backupManifestPath

	// Skip upgrade if current and new manifests are equal
	equal, err := staticpod.ManifestFilesAreEqual(currentManifestPath, newManifestPath)
	if err != nil {
		return err
	}
	if equal {
		fmt.Printf("[upgrade/staticpods] Current and new manifests of %s are equal, skipping upgrade\n", component)
		return nil
	}

	// if certificate renewal should be performed
	if certsRenewMgr != nil {
		// renew all the certificates used by the current component
		if err := renewCertsByComponent(cfg, component, certsRenewMgr); err != nil {
			return rollbackOldManifests(recoverManifests, errors.Wrapf(err, "failed to renew certificates for component %q", component), pathMgr, recoverEtcd)
		}
	}

	// Move the old manifest into the old-manifests directory
	if err := pathMgr.MoveFile(currentManifestPath, backupManifestPath); err != nil {
		return rollbackOldManifests(recoverManifests, err, pathMgr, recoverEtcd)
	}

	// Move the new manifest into the manifests directory
	if err := pathMgr.MoveFile(newManifestPath, currentManifestPath); err != nil {
		return rollbackOldManifests(recoverManifests, err, pathMgr, recoverEtcd)
	}

	fmt.Printf("[upgrade/staticpods] Moved new manifest to %q and backed up old manifest to %q\n", currentManifestPath, backupManifestPath)

	fmt.Println("[upgrade/staticpods] Waiting for the kubelet to restart the component")
	fmt.Printf("[upgrade/staticpods] This might take a minute or longer depending on the component/version gap (timeout %v)\n", UpgradeManifestTimeout)

	// Wait for the mirror Pod hash to change; otherwise we'll run into race conditions here when the kubelet hasn't had time to
	// notice the removal of the Static Pod, leading to a false positive below where we check that the API endpoint is healthy
	// If we don't do this, there is a case where we remove the Static Pod manifest, kubelet is slow to react, kubeadm checks the
	// API endpoint below of the OLD Static Pod component and proceeds quickly enough, which might lead to unexpected results.
	if err := waiter.WaitForStaticPodHashChange(cfg.NodeRegistration.Name, component, beforePodHash); err != nil {
		return rollbackOldManifests(recoverManifests, err, pathMgr, recoverEtcd)
	}

	// Wait for the static pod component to come up and register itself as a mirror pod
	if err := waiter.WaitForPodsWithLabel("component=" + component); err != nil {
		return rollbackOldManifests(recoverManifests, err, pathMgr, recoverEtcd)
	}

	fmt.Printf("[upgrade/staticpods] Component %q upgraded successfully!\n", component)

	return nil
}


performEtcdStaticPodUpgrade

// performEtcdStaticPodUpgrade performs upgrade of etcd, it returns bool which indicates fatal error or not and the actual error.
func performEtcdStaticPodUpgrade(certsRenewMgr *renewal.Manager, client clientset.Interface, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, recoverManifests map[string]string, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) (bool, error) {
	// Add etcd static pod spec only if external etcd is not configured
	if cfg.Etcd.External != nil {
		return false, errors.New("external etcd detected, won't try to change any etcd state")
	}

	// Checking health state of etcd before proceeding with the upgrade
	err := oldEtcdClient.CheckClusterHealth()
	if err != nil {
		return true, errors.Wrap(err, "etcd cluster is not healthy")
	}

	// Backing up etcd data store
	backupEtcdDir := pathMgr.BackupEtcdDir()
	runningEtcdDir := cfg.Etcd.Local.DataDir
	output, err := kubeadmutil.CopyDir(runningEtcdDir, backupEtcdDir)
	if err != nil {
		return true, errors.Wrapf(err, "failed to back up etcd data, output: %q", output)
	}

	// Get the desired etcd version. That's either the one specified by the user in cfg.Etcd.Local.ImageTag
	// or the kubeadm preferred one for the desired Kubernetes version
	var desiredEtcdVersion *version.Version
	if cfg.Etcd.Local.ImageTag != "" {
		desiredEtcdVersion, err = version.ParseSemantic(
			convertImageTagMetadataToSemver(cfg.Etcd.Local.ImageTag))
		if err != nil {
			return true, errors.Wrapf(err, "failed to parse tag %q as a semantic version", cfg.Etcd.Local.ImageTag)
		}
	} else {
		// Need to check currently used version and version from constants, if differs then upgrade
		var warning error
		desiredEtcdVersion, warning, err = constants.EtcdSupportedVersion(constants.SupportedEtcdVersion, cfg.KubernetesVersion)
		if err != nil {
			return true, errors.Wrap(err, "failed to retrieve an etcd version for the target Kubernetes version")
		}
		if warning != nil {
			klog.Warningf("[upgrade/etcd] %v", warning)
		}
	}

	// Get the etcd version of the local/stacked etcd member running on the current machine
	currentEtcdVersionStr, err := GetEtcdImageTagFromStaticPod(pathMgr.RealManifestDir())
	if err != nil {
		return true, errors.Wrap(err, "failed to retrieve the current etcd version")
	}

	cmpResult, err := desiredEtcdVersion.Compare(currentEtcdVersionStr)
	if err != nil {
		return true, errors.Wrapf(err, "failed comparing the current etcd version %q to the desired one %q", currentEtcdVersionStr, desiredEtcdVersion)
	}
	if cmpResult < 0 {
		return false, errors.Errorf("the desired etcd version %q is older than the currently installed %q. Skipping etcd upgrade", desiredEtcdVersion, currentEtcdVersionStr)
	}

	beforeEtcdPodHash, err := waiter.WaitForStaticPodSingleHash(cfg.NodeRegistration.Name, constants.Etcd)
	if err != nil {
		return true, err
	}

	// Write the updated etcd static Pod manifest into the temporary directory, at this point no etcd change
	// has occurred in any aspects.
	if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.TempManifestDir(), pathMgr.PatchesDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, false /* isDryRun */); err != nil {
		return true, errors.Wrap(err, "error creating local etcd static pod manifest file")
	}

	retries := 10
	retryInterval := 15 * time.Second

	// Perform etcd upgrade using common to all control plane components function
	if err := upgradeComponent(constants.Etcd, certsRenewMgr, waiter, pathMgr, cfg, beforeEtcdPodHash, recoverManifests); err != nil {
		fmt.Printf("[upgrade/etcd] Failed to upgrade etcd: %v\n", err)
		// Since upgrade component failed, the old etcd manifest has either been restored or was never touched
		// Now we need to check the health of etcd cluster if it is up with old manifest
		fmt.Println("[upgrade/etcd] Waiting for previous etcd to become available")
		if _, err := oldEtcdClient.WaitForClusterAvailable(retries, retryInterval); err != nil {
			fmt.Printf("[upgrade/etcd] Failed to healthcheck previous etcd: %v\n", err)

			// At this point we know that etcd cluster is dead and it is safe to copy backup datastore and to rollback old etcd manifest
			fmt.Println("[upgrade/etcd] Rolling back etcd data")
			if err := rollbackEtcdData(cfg, pathMgr); err != nil {
				// Even copying back datastore failed, no options for recovery left, bailing out
				return true, errors.Errorf("fatal error rolling back local etcd cluster datadir: %v, the backup of etcd database is stored here:(%s)", err, backupEtcdDir)
			}
			fmt.Println("[upgrade/etcd] Etcd data rollback successful")

			// Now that we've rolled back the data, let's check if the cluster comes up
			fmt.Println("[upgrade/etcd] Waiting for previous etcd to become available")
			if _, err := oldEtcdClient.WaitForClusterAvailable(retries, retryInterval); err != nil {
				fmt.Printf("[upgrade/etcd] Failed to healthcheck previous etcd: %v\n", err)
				// Nothing else left to try to recover etcd cluster
				return true, errors.Wrapf(err, "fatal error rolling back local etcd cluster manifest, the backup of etcd database is stored here:(%s)", backupEtcdDir)
			}

			// We've recovered to the previous etcd from this case
		}
		fmt.Println("[upgrade/etcd] Etcd was rolled back and is now available")

		// Since etcd cluster came back up with the old manifest
		return true, errors.Wrap(err, "fatal error when trying to upgrade the etcd cluster, rolled the state back to pre-upgrade state")
	}

	// Initialize the new etcd client if it wasn't pre-initialized
	if newEtcdClient == nil {
		etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir)
		if err != nil {
			return true, errors.Wrap(err, "fatal error creating etcd client")
		}
		newEtcdClient = etcdClient
	}

	// Checking health state of etcd after the upgrade
	fmt.Println("[upgrade/etcd] Waiting for etcd to become available")
	if _, err = newEtcdClient.WaitForClusterAvailable(retries, retryInterval); err != nil {
		fmt.Printf("[upgrade/etcd] Failed to healthcheck etcd: %v\n", err)
		// Despite the fact that upgradeComponent was successful, there is something wrong with the etcd cluster
		// First step is to restore back up of datastore
		fmt.Println("[upgrade/etcd] Rolling back etcd data")
		if err := rollbackEtcdData(cfg, pathMgr); err != nil {
			// Even copying back datastore failed, no options for recovery left, bailing out
			return true, errors.Wrapf(err, "fatal error rolling back local etcd cluster datadir, the backup of etcd database is stored here:(%s)", backupEtcdDir)
		}
		fmt.Println("[upgrade/etcd] Etcd data rollback successful")

		// Old datastore has been copied, rolling back old manifests
		fmt.Println("[upgrade/etcd] Rolling back etcd manifest")
		rollbackOldManifests(recoverManifests, err, pathMgr, true)
		// rollbackOldManifests() always returns an error -- ignore it and continue

		// Assuming rollback of the old etcd manifest was successful, check the status of etcd cluster again
		fmt.Println("[upgrade/etcd] Waiting for previous etcd to become available")
		if _, err := oldEtcdClient.WaitForClusterAvailable(retries, retryInterval); err != nil {
			fmt.Printf("[upgrade/etcd] Failed to healthcheck previous etcd: %v\n", err)
			// Nothing else left to try to recover etcd cluster
			return true, errors.Wrapf(err, "fatal error rolling back local etcd cluster manifest, the backup of etcd database is stored here:(%s)", backupEtcdDir)
		}
		fmt.Println("[upgrade/etcd] Etcd was rolled back and is now available")

		// We've successfully rolled back etcd, and now return an error describing that the upgrade failed
		return true, errors.Wrap(err, "fatal error upgrading local etcd cluster, rolled the state back to pre-upgrade state")
	}

	return false, nil
}


StaticPodControlPlane

// StaticPodControlPlane upgrades a static pod-hosted control plane
func StaticPodControlPlane(client clientset.Interface, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, etcdUpgrade, renewCerts bool, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) error {
	recoverManifests := map[string]string{}
	var isExternalEtcd bool

	beforePodHashMap, err := waiter.WaitForStaticPodControlPlaneHashes(cfg.NodeRegistration.Name)
	if err != nil {
		return err
	}

	if oldEtcdClient == nil {
		if cfg.Etcd.External != nil {
			// External etcd
			isExternalEtcd = true
			etcdClient, err := etcdutil.New(
				cfg.Etcd.External.Endpoints,
				cfg.Etcd.External.CAFile,
				cfg.Etcd.External.CertFile,
				cfg.Etcd.External.KeyFile,
			)
			if err != nil {
				return errors.Wrap(err, "failed to create etcd client for external etcd")
			}
			oldEtcdClient = etcdClient
			// Since etcd is managed externally, the new etcd client will be the same as the old client
			if newEtcdClient == nil {
				newEtcdClient = etcdClient
			}
		} else {
			// etcd Static Pod
			etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir)
			if err != nil {
				return errors.Wrap(err, "failed to create etcd client")
			}
			oldEtcdClient = etcdClient
		}
	}

	var certsRenewMgr *renewal.Manager
	if renewCerts {
		certsRenewMgr, err = renewal.NewManager(&cfg.ClusterConfiguration, pathMgr.KubernetesDir())
		if err != nil {
			return errors.Wrap(err, "failed to create the certificate renewal manager")
		}
	}

	// etcd upgrade is done prior to other control plane components
	if !isExternalEtcd && etcdUpgrade {
		// set the TLS upgrade flag for all components
		fmt.Printf("[upgrade/etcd] Upgrading to TLS for %s\n", constants.Etcd)

		// Perform etcd upgrade using common to all control plane components function
		fatal, err := performEtcdStaticPodUpgrade(certsRenewMgr, client, waiter, pathMgr, cfg, recoverManifests, oldEtcdClient, newEtcdClient)
		if err != nil {
			if fatal {
				return err
			}
			fmt.Printf("[upgrade/etcd] Non fatal issue encountered during upgrade: %v\n", err)
		}
	}

	// Write the updated static Pod manifests into the temporary directory
	fmt.Printf("[upgrade/staticpods] Writing new Static Pod manifests to %q\n", pathMgr.TempManifestDir())
	err = controlplane.CreateInitStaticPodManifestFiles(pathMgr.TempManifestDir(), pathMgr.PatchesDir(), cfg, false /* isDryRun */)
	if err != nil {
		return errors.Wrap(err, "error creating init static pod manifest files")
	}

	for _, component := range constants.ControlPlaneComponents {
		if err = upgradeComponent(component, certsRenewMgr, waiter, pathMgr, cfg, beforePodHashMap[component], recoverManifests); err != nil {
			return err
		}
	}

	if renewCerts {
		// renew the certificate embedded in the admin.conf file
		renewed, err := certsRenewMgr.RenewUsingLocalCA(constants.AdminKubeConfigFileName)
		if err != nil {
			return rollbackOldManifests(recoverManifests, errors.Wrapf(err, "failed to upgrade the %s certificates", constants.AdminKubeConfigFileName), pathMgr, false)
		}

		if !renewed {
			// if not error, but not renewed because of external CA detected, inform the user
			fmt.Printf("[upgrade/staticpods] External CA detected, %s certificate can't be renewed\n", constants.AdminKubeConfigFileName)
		}
	}

	// Remove the temporary directories used on a best-effort (don't fail if the calls error out)
	// The calls are set here by design; we should _not_ use "defer" above as that would remove the directories
	// even in the "fail and rollback" case, where we want the directories preserved for the user.
	return pathMgr.CleanupDirs()
}


rollbackOldManifests

// rollbackOldManifests rolls back the backed-up manifests if something went wrong.
// It always returns an error to the caller.
func rollbackOldManifests(oldManifests map[string]string, origErr error, pathMgr StaticPodPathManager, restoreEtcd bool) error {
	errs := []error{origErr}
	for component, backupPath := range oldManifests {
		// Will restore etcd manifest only if it was explicitly requested by setting restoreEtcd to True
		if component == constants.Etcd && !restoreEtcd {
			continue
		}
		// Where we should put back the backed up manifest
		realManifestPath := pathMgr.RealManifestPath(component)

		// Move the backup manifest back into the manifests directory
		err := pathMgr.MoveFile(backupPath, realManifestPath)
		if err != nil {
			errs = append(errs, err)
		}
	}
	// Let the user know there were problems, but we tried to recover
	return errors.Wrap(utilerrors.NewAggregate(errs),
		"couldn't upgrade control plane. kubeadm has tried to recover everything into the earlier state. Errors faced")
}


rollbackEtcdData

// rollbackEtcdData rolls back the content of etcd folder if something went wrong.
// When the folder contents are successfully rolled back, nil is returned, otherwise an error is returned.
func rollbackEtcdData(cfg *kubeadmapi.InitConfiguration, pathMgr StaticPodPathManager) error {
	backupEtcdDir := pathMgr.BackupEtcdDir()
	runningEtcdDir := cfg.Etcd.Local.DataDir

	output, err := kubeadmutil.CopyDir(backupEtcdDir, runningEtcdDir)
	if err != nil {
		// Let the user know there we're problems, but we tried to reçover
		return errors.Wrapf(err, "couldn't recover etcd database with error, the location of etcd backup: %s, output: %q", backupEtcdDir, output)
	}

	return nil
}


renewCertsByComponent

// renewCertsByComponent takes charge of renewing certificates used by a specific component before
// the static pod of the component is upgraded
func renewCertsByComponent(cfg *kubeadmapi.InitConfiguration, component string, certsRenewMgr *renewal.Manager) error {
	var certificates []string

	// if etcd, only in case of local etcd, renew server, peer and health check certificate
	if component == constants.Etcd {
		if cfg.Etcd.Local != nil {
			certificates = []string{
				certsphase.KubeadmCertEtcdServer().Name,
				certsphase.KubeadmCertEtcdPeer().Name,
				certsphase.KubeadmCertEtcdHealthcheck().Name,
			}
		}
	}

	// if apiserver, renew apiserver serving certificate, kubelet and front-proxy client certificate.
	//if local etcd, renew also the etcd client certificate
	if component == constants.KubeAPIServer {
		certificates = []string{
			certsphase.KubeadmCertAPIServer().Name,
			certsphase.KubeadmCertKubeletClient().Name,
			certsphase.KubeadmCertFrontProxyClient().Name,
		}
		if cfg.Etcd.Local != nil {
			certificates = append(certificates, certsphase.KubeadmCertEtcdAPIClient().Name)
		}
	}

	// if controller-manager, renew the certificate embedded in the controller-manager kubeConfig file
	if component == constants.KubeControllerManager {
		certificates = []string{
			constants.ControllerManagerKubeConfigFileName,
		}
	}

	// if scheduler, renew the certificate embedded in the scheduler kubeConfig file
	if component == constants.KubeScheduler {
		certificates = []string{
			constants.SchedulerKubeConfigFileName,
		}
	}

	// renew the selected components
	for _, cert := range certificates {
		fmt.Printf("[upgrade/staticpods] Renewing %s certificate\n", cert)
		renewed, err := certsRenewMgr.RenewUsingLocalCA(cert)
		if err != nil {
			return err
		}
		if !renewed {
			// if not error, but not renewed because of external CA detected, inform the user
			fmt.Printf("[upgrade/staticpods] External CA detected, %s certificate can't be renewed\n", cert)
		}
	}

	return nil
}


GetPathManagerForUpgrade

// GetPathManagerForUpgrade returns a path manager properly configured for the given InitConfiguration.
func GetPathManagerForUpgrade(kubernetesDir, patchesDir string, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade bool) (StaticPodPathManager, error) {
	isExternalEtcd := internalcfg.Etcd.External != nil
	return NewKubeStaticPodPathManagerUsingTempDirs(kubernetesDir, patchesDir, true, etcdUpgrade && !isExternalEtcd)
}


PerformStaticPodUpgrade

// PerformStaticPodUpgrade performs the upgrade of the control plane components for a static pod hosted cluster
func PerformStaticPodUpgrade(client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade, renewCerts bool, patchesDir string) error {
	pathManager, err := GetPathManagerForUpgrade(constants.KubernetesDir, patchesDir, internalcfg, etcdUpgrade)
	if err != nil {
		return err
	}

	// The arguments oldEtcdClient and newEtdClient, are uninitialized because passing in the clients allow for mocking the client during testing
	return StaticPodControlPlane(client, waiter, pathManager, internalcfg, etcdUpgrade, renewCerts, nil, nil)
}


DryRunStaticPodUpgrade

// DryRunStaticPodUpgrade fakes an upgrade of the control plane
func DryRunStaticPodUpgrade(patchesDir string, internalcfg *kubeadmapi.InitConfiguration) error {

	dryRunManifestDir, err := constants.CreateTempDirForKubeadm("", "kubeadm-upgrade-dryrun")
	if err != nil {
		return err
	}
	defer os.RemoveAll(dryRunManifestDir)
	if err := controlplane.CreateInitStaticPodManifestFiles(dryRunManifestDir, patchesDir, internalcfg, true /* isDryRun */); err != nil {
		return err
	}

	// Print the contents of the upgraded manifests and pretend like they were in /etc/kubernetes/manifests
	files := []dryrunutil.FileToPrint{}
	for _, component := range constants.ControlPlaneComponents {
		realPath := constants.GetStaticPodFilepath(component, dryRunManifestDir)
		outputPath := constants.GetStaticPodFilepath(component, constants.GetStaticPodDirectory())
		files = append(files, dryrunutil.NewFileToPrint(realPath, outputPath))
	}

	return dryrunutil.PrintDryRunFiles(files, os.Stdout)
}


GetEtcdImageTagFromStaticPod

// GetEtcdImageTagFromStaticPod returns the image tag of the local etcd static pod
func GetEtcdImageTagFromStaticPod(manifestDir string) (string, error) {
	realPath := constants.GetStaticPodFilepath(constants.Etcd, manifestDir)
	pod, err := staticpod.ReadStaticPodFromDisk(realPath)
	if err != nil {
		return "", err
	}

	return convertImageTagMetadataToSemver(image.TagFromImage(pod.Spec.Containers[0].Image)), nil
}


convertImageTagMetadataToSemver

// convertImageTagMetadataToSemver converts imagetag in the format of semver_metadata to semver+metadata
func convertImageTagMetadataToSemver(tag string) string {
	// Container registries do not support `+` characters in tag names. This prevents imagetags from
	// correctly representing semantic versions which use the plus symbol to delimit build metadata.
	// Kubernetes uses the convention of using an underscore in image registries to preserve
	// build metadata information in imagetags.
	return strings.Replace(tag, "_", "+", 1)
}


NewKubeVersionGetter

// NewKubeVersionGetter returns a new instance of KubeVersionGetter
func NewKubeVersionGetter(client clientset.Interface) VersionGetter {
	return &KubeVersionGetter{
		client: client,
	}
}


ClusterVersion

// ClusterVersion gets API server version
func (g *KubeVersionGetter) ClusterVersion() (string, *versionutil.Version, error) {
	var (
		clusterVersionInfo *pkgversion.Info
		err                error
	)
	// If we are dry-running, do not attempt to fetch the /version resource and just return
	// the stored FakeServerVersion, which is done when constructing the dry-run client in
	// common.go#getClient()
	// The problem here is that during upgrade dry-run client reactors are backed by a dynamic client
	// via NewClientBackedDryRunGetterFromKubeconfig() and for GetActions there seems to be no analog to
	// Discovery().Serverversion() resource for a dynamic client(?).
	fakeclientDiscovery, ok := g.client.Discovery().(*fakediscovery.FakeDiscovery)
	if ok {
		clusterVersionInfo = fakeclientDiscovery.FakedServerVersion
	} else {
		clusterVersionInfo, err = g.client.Discovery().ServerVersion()
		if err != nil {
			return "", nil, errors.Wrap(err, "Couldn't fetch cluster version from the API Server")
		}
	}

	clusterVersion, err := versionutil.ParseSemantic(clusterVersionInfo.String())
	if err != nil {
		return "", nil, errors.Wrap(err, "Couldn't parse cluster version")
	}
	return clusterVersionInfo.String(), clusterVersion, nil
}


KubeadmVersion

// KubeadmVersion gets kubeadm version
func (g *KubeVersionGetter) KubeadmVersion() (string, *versionutil.Version, error) {
	kubeadmVersionInfo := version.Get()

	kubeadmVersion, err := versionutil.ParseSemantic(kubeadmVersionInfo.String())
	if err != nil {
		return "", nil, errors.Wrap(err, "Couldn't parse kubeadm version")
	}
	return kubeadmVersionInfo.String(), kubeadmVersion, nil
}


VersionFromCILabel

// VersionFromCILabel resolves a version label like "latest" or "stable" to an actual version using the public Kubernetes CI uploads
func (g *KubeVersionGetter) VersionFromCILabel(ciVersionLabel, description string) (string, *versionutil.Version, error) {
	versionStr, err := kubeadmutil.KubernetesReleaseVersion(ciVersionLabel)
	if err != nil {
		return "", nil, errors.Wrapf(err, "Couldn't fetch latest %s from the internet", description)
	}

	ver, err := versionutil.ParseSemantic(versionStr)
	if err != nil {
		return "", nil, errors.Wrapf(err, "Couldn't parse latest %s", description)
	}
	return versionStr, ver, nil
}


KubeletVersions

// KubeletVersions gets the versions of the kubelets in the cluster
func (g *KubeVersionGetter) KubeletVersions() (map[string]uint16, error) {
	nodes, err := g.client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
	if err != nil {
		return nil, errors.New("couldn't list all nodes in cluster")
	}
	return computeKubeletVersions(nodes.Items), nil
}


computeKubeletVersions

// computeKubeletVersions returns a string-int map that describes how many nodes are of a specific version
func computeKubeletVersions(nodes []v1.Node) map[string]uint16 {
	kubeletVersions := make(map[string]uint16)
	for _, node := range nodes {
		kver := node.Status.NodeInfo.KubeletVersion
		if _, found := kubeletVersions[kver]; !found {
			kubeletVersions[kver] = 1
			continue
		}
		kubeletVersions[kver]++
	}
	return kubeletVersions
}


NewOfflineVersionGetter

// NewOfflineVersionGetter wraps a VersionGetter and skips online communication if default information is supplied.
// Version can be "" and the behavior will be identical to the versionGetter passed in.
func NewOfflineVersionGetter(versionGetter VersionGetter, version string) VersionGetter {
	return &OfflineVersionGetter{
		VersionGetter: versionGetter,
		version:       version,
	}
}


VersionFromCILabel

// VersionFromCILabel will return the version that was passed into the struct
func (o *OfflineVersionGetter) VersionFromCILabel(ciVersionLabel, description string) (string, *versionutil.Version, error) {
	if o.version == "" {
		return o.VersionGetter.VersionFromCILabel(ciVersionLabel, description)
	}
	ver, err := versionutil.ParseSemantic(o.version)
	if err != nil {
		return "", nil, errors.Wrapf(err, "Couldn't parse version %s", description)
	}
	return o.version, ver, nil
}


UploadConfiguration

// UploadConfiguration saves the InitConfiguration used for later reference (when upgrading for instance)
func UploadConfiguration(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error {
	fmt.Printf("[upload-config] Storing the configuration used in ConfigMap %q in the %q Namespace\n", kubeadmconstants.KubeadmConfigConfigMap, metav1.NamespaceSystem)

	// Prepare the ClusterConfiguration for upload
	// The components store their config in their own ConfigMaps, then reset the .ComponentConfig struct;
	// We don't want to mutate the cfg itself, so create a copy of it using .DeepCopy of it first
	clusterConfigurationToUpload := cfg.ClusterConfiguration.DeepCopy()
	clusterConfigurationToUpload.ComponentConfigs = kubeadmapi.ComponentConfigMap{}

	// restore the resolved Kubernetes version as CI Kubernetes version if needed
	if len(clusterConfigurationToUpload.CIKubernetesVersion) > 0 {
		clusterConfigurationToUpload.KubernetesVersion = clusterConfigurationToUpload.CIKubernetesVersion
	}

	// Marshal the ClusterConfiguration into YAML
	clusterConfigurationYaml, err := configutil.MarshalKubeadmConfigObject(clusterConfigurationToUpload)
	if err != nil {
		return err
	}

	err = apiclient.CreateOrMutateConfigMap(client, &v1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Name:      kubeadmconstants.KubeadmConfigConfigMap,
			Namespace: metav1.NamespaceSystem,
		},
		Data: map[string]string{
			kubeadmconstants.ClusterConfigurationConfigMapKey: string(clusterConfigurationYaml),
		},
	}, func(cm *v1.ConfigMap) error {
		// Upgrade will call to UploadConfiguration with a modified KubernetesVersion reflecting the new
		// Kubernetes version. In that case, the mutation path will take place.
		cm.Data[kubeadmconstants.ClusterConfigurationConfigMapKey] = string(clusterConfigurationYaml)
		return nil
	})
	if err != nil {
		return err
	}

	// Ensure that the NodesKubeadmConfigClusterRoleName exists
	err = apiclient.CreateOrUpdateRole(client, &rbac.Role{
		ObjectMeta: metav1.ObjectMeta{
			Name:      NodesKubeadmConfigClusterRoleName,
			Namespace: metav1.NamespaceSystem,
		},
		Rules: []rbac.PolicyRule{
			{
				Verbs:         []string{"get"},
				APIGroups:     []string{""},
				Resources:     []string{"configmaps"},
				ResourceNames: []string{kubeadmconstants.KubeadmConfigConfigMap},
			},
		},
	})
	if err != nil {
		return err
	}

	// Binds the NodesKubeadmConfigClusterRoleName to all the bootstrap tokens
	// that are members of the system:bootstrappers:kubeadm:default-node-token group
	// and to all nodes
	return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{
		ObjectMeta: metav1.ObjectMeta{
			Name:      NodesKubeadmConfigClusterRoleName,
			Namespace: metav1.NamespaceSystem,
		},
		RoleRef: rbac.RoleRef{
			APIGroup: rbac.GroupName,
			Kind:     "Role",
			Name:     NodesKubeadmConfigClusterRoleName,
		},
		Subjects: []rbac.Subject{
			{
				Kind: rbac.GroupKind,
				Name: kubeadmconstants.NodeBootstrapTokenAuthGroup,
			},
			{
				Kind: rbac.GroupKind,
				Name: kubeadmconstants.NodesGroup,
			},
		},
	})
}


Error

// Error implements the standard error interface
func (e *Error) Error() string {
	return fmt.Sprintf("[preflight] Some fatal errors occurred:\n%s%s", e.Msg, "[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`")
}


Preflight

// Preflight identifies this error as a preflight error
func (e *Error) Preflight() bool {
	return true
}


Name

// Name returns label for RuntimeCheck.
func (ContainerRuntimeCheck) Name() string {
	return "CRI"
}


Check

// Check validates the container runtime
func (crc ContainerRuntimeCheck) Check() (warnings, errorList []error) {
	klog.V(1).Infoln("validating the container runtime")
	if err := crc.runtime.IsRunning(); err != nil {
		errorList = append(errorList, err)
	}
	return warnings, errorList
}