// Run the kubeadm composable kubeadm workflows.
func (e *Runner) Run(args []string) error {
	e.prepareForExecution()

	// determine which phase should be run according to RunnerOptions
	phaseRunFlags, err := e.computePhaseRunFlags()
	if err != nil {
		return err
	}

	// precheck phase dependencies before actual execution
	missedDeps := make(map[string][]string)
	visited := make(map[string]struct{})
	for _, p := range e.phaseRunners {
		if run, ok := phaseRunFlags[p.generatedName]; !run || !ok {
			continue
		}
		for _, dep := range p.Phase.Dependencies {
			if _, ok := visited[dep]; !ok {
				missedDeps[p.Phase.Name] = append(missedDeps[p.Phase.Name], dep)
			}
		}
		visited[p.Phase.Name] = struct{}{}
	}
	if len(missedDeps) > 0 {
		var msg strings.Builder
		msg.WriteString("unresolved dependencies:")
		for phase, missedPhases := range missedDeps {
			msg.WriteString(fmt.Sprintf("\n\tmissing %v phase(s) needed by %q phase", missedPhases, phase))
		}
		return errors.New(msg.String())
	}

	// builds the runner data if the runtime data is not initialized
	data := e.runData
	if data == nil {
		if data, err = e.InitData(args); err != nil {
			return err
		}
	}

	err = e.visitAll(func(p *phaseRunner) error {
		// if the phase should not be run, skip the phase.
		if run, ok := phaseRunFlags[p.generatedName]; !run || !ok {
			return nil
		}

		// Errors if phases that are meant to create special subcommands only
		// are wrongly assigned Run Methods
		if p.RunAllSiblings && (p.RunIf != nil || p.Run != nil) {
			return errors.Errorf("phase marked as RunAllSiblings can not have Run functions %s", p.generatedName)
		}

		// If the phase defines a condition to be checked before executing the phase action.
		if p.RunIf != nil {
			// Check the condition and returns if the condition isn't satisfied (or fails)
			ok, err := p.RunIf(data)
			if err != nil {
				return errors.Wrapf(err, "error execution run condition for phase %s", p.generatedName)
			}

			if !ok {
				return nil
			}
		}

		// Runs the phase action (if defined)
		if p.Run != nil {
			if err := p.Run(data); err != nil {
				return errors.Wrapf(err, "error execution phase %s", p.generatedName)
			}
		}

		return nil
	})

	return err
}


// Help returns text with the list of phases included in the workflow.
func (e *Runner) Help(cmdUse string) string {
	e.prepareForExecution()

	// computes the max length of for each phase use line
	maxLength := 0
	e.visitAll(func(p *phaseRunner) error {
		if !p.Hidden && !p.RunAllSiblings {
			length := len(p.use)
			if maxLength < length {
				maxLength = length
			}
		}
		return nil
	})

	// prints the list of phases indented by level and formatted using the maxlength
	// the list is enclosed in a mardown code block for ensuring better readability in the public web site
	line := fmt.Sprintf("The %q command executes the following phases:\n", cmdUse)
	line += "```\n"
	offset := 2
	e.visitAll(func(p *phaseRunner) error {
		if !p.Hidden && !p.RunAllSiblings {
			padding := maxLength - len(p.use) + offset
			line += strings.Repeat(" ", offset*p.level) // indentation
			line += p.use                               // name + aliases
			line += strings.Repeat(" ", padding)        // padding right up to max length (+ offset for spacing)
			line += p.Short                             // phase short description
			line += "\n"
		}

		return nil
	})
	line += "```"
	return line
}


// SetAdditionalFlags allows to define flags to be added
// to the subcommands generated for each phase (but not existing in the parent command).
// Please note that this command needs to be done before BindToCommand.
// Nb. if a flag is used only by one phase, please consider using phase LocalFlags.
func (e *Runner) SetAdditionalFlags(fn func(*pflag.FlagSet)) {
	// creates a new NewFlagSet
	e.cmdAdditionalFlags = pflag.NewFlagSet("phaseAdditionalFlags", pflag.ContinueOnError)
	// invokes the function that sets additional flags
	fn(e.cmdAdditionalFlags)
}


// BindToCommand bind the Runner to a cobra command by altering
// command help, adding phase related flags and by adding phases subcommands
// Please note that this command needs to be done once all the phases are added to the Runner.
func (e *Runner) BindToCommand(cmd *cobra.Command) {
	// keep track of the command triggering the runner
	e.runCmd = cmd

	// return early if no phases were added
	if len(e.Phases) == 0 {
		return
	}

	e.prepareForExecution()

	// adds the phases subcommand
	phaseCommand := &cobra.Command{
		Use:   "phase",
		Short: fmt.Sprintf("Use this command to invoke single phase of the %s workflow", cmd.Name()),
	}

	cmd.AddCommand(phaseCommand)

	// generate all the nested subcommands for invoking single phases
	subcommands := map[string]*cobra.Command{}
	e.visitAll(func(p *phaseRunner) error {
		// skip hidden phases
		if p.Hidden {
			return nil
		}

		// initialize phase selector
		phaseSelector := p.generatedName

		// if requested, set the phase to run all the sibling phases
		if p.RunAllSiblings {
			phaseSelector = p.parent.generatedName
		}

		// creates phase subcommand
		phaseCmd := &cobra.Command{
			Use:     strings.ToLower(p.Name),
			Short:   p.Short,
			Long:    p.Long,
			Example: p.Example,
			Aliases: p.Aliases,
			RunE: func(cmd *cobra.Command, args []string) error {
				// if the phase has subphases, print the help and exits
				if len(p.Phases) > 0 {
					return cmd.Help()
				}

				// overrides the command triggering the Runner using the phaseCmd
				e.runCmd = cmd
				e.Options.FilterPhases = []string{phaseSelector}
				return e.Run(args)
			},
		}

		// makes the new command inherits local flags from the parent command
		// Nb. global flags will be inherited automatically
		inheritsFlags(cmd.Flags(), phaseCmd.Flags(), p.InheritFlags)

		// makes the new command inherits additional flags for phases
		if e.cmdAdditionalFlags != nil {
			inheritsFlags(e.cmdAdditionalFlags, phaseCmd.Flags(), p.InheritFlags)
		}

		// If defined, added phase local flags
		if p.LocalFlags != nil {
			p.LocalFlags.VisitAll(func(f *pflag.Flag) {
				phaseCmd.Flags().AddFlag(f)
			})
		}

		// if this phase has children (not a leaf) it doesn't accept any args
		if len(p.Phases) > 0 {
			phaseCmd.Args = cobra.NoArgs
		} else {
			if p.ArgsValidator == nil {
				phaseCmd.Args = cmd.Args
			} else {
				phaseCmd.Args = p.ArgsValidator
			}
		}

		// adds the command to parent
		if p.level == 0 {
			phaseCommand.AddCommand(phaseCmd)
		} else {
			subcommands[p.parent.generatedName].AddCommand(phaseCmd)
		}

		subcommands[p.generatedName] = phaseCmd
		return nil
	})

	// alters the command description to show available phases
	if cmd.Long != "" {
		cmd.Long = fmt.Sprintf("%s\n\n%s\n", cmd.Long, e.Help(cmd.Use))
	} else {
		cmd.Long = fmt.Sprintf("%s\n\n%s\n", cmd.Short, e.Help(cmd.Use))
	}

	// adds phase related flags to the main command
	cmd.Flags().StringSliceVar(&e.Options.SkipPhases, "skip-phases", nil, "List of phases to be skipped")
}


func inheritsFlags(sourceFlags, targetFlags *pflag.FlagSet, cmdFlags []string) {
	// If the list of flag to be inherited from the parent command is not defined, no flag is added
	if cmdFlags == nil {
		return
	}

	// add all the flags to be inherited to the target flagSet
	sourceFlags.VisitAll(func(f *pflag.Flag) {
		for _, c := range cmdFlags {
			if f.Name == c {
				targetFlags.AddFlag(f)
			}
		}
	})
}


// visitAll provides a utility method for visiting all the phases in the workflow
// in the execution order and executing a func on each phase.
// Nested phase are visited immediately after their parent phase.
func (e *Runner) visitAll(fn func(*phaseRunner) error) error {
	for _, currentRunner := range e.phaseRunners {
		if err := fn(currentRunner); err != nil {
			return err
		}
	}
	return nil
}


func (e *Runner) prepareForExecution()

// prepareForExecution initialize the internal state of the Runner (the list of phaseRunner).
func (e *Runner) prepareForExecution() {
	e.phaseRunners = []*phaseRunner{}
	var parentRunner *phaseRunner
	for _, phase := range e.Phases {
		// skips phases that are meant to create special subcommands only
		if phase.RunAllSiblings {
			continue
		}

		// add phases to the execution list
		addPhaseRunner(e, parentRunner, phase)
	}
}


// addPhaseRunner adds the phaseRunner for a given phase to the phaseRunners list
func addPhaseRunner(e *Runner, parentRunner *phaseRunner, phase Phase) {
	// computes contextual information derived by the workflow managed by the Runner.
	use := cleanName(phase.Name)
	generatedName := use
	selfPath := []string{generatedName}

	if parentRunner != nil {
		generatedName = strings.Join([]string{parentRunner.generatedName, generatedName}, phaseSeparator)
		use = fmt.Sprintf("%s%s", phaseSeparator, use)
		selfPath = append(parentRunner.selfPath, selfPath...)
	}

	// creates the phaseRunner
	currentRunner := &phaseRunner{
		Phase:         phase,
		parent:        parentRunner,
		level:         len(selfPath) - 1,
		selfPath:      selfPath,
		generatedName: generatedName,
		use:           use,
	}

	// adds to the phaseRunners list
	e.phaseRunners = append(e.phaseRunners, currentRunner)

	// iterate for the nested, ordered list of phases, thus storing
	// phases in the expected executing order (child phase are stored immediately after their parent phase).
	for _, childPhase := range phase.Phases {
		addPhaseRunner(e, currentRunner, childPhase)
	}
}


func cleanName(name string) string

// cleanName makes phase name suitable for the runner help, by lowercasing the name
// and removing args descriptors, if any
func cleanName(name string) string {
	ret := strings.ToLower(name)
	if pos := strings.Index(ret, " "); pos != -1 {
		ret = ret[:pos]
	}
	return ret
}


func newResetOptions() *resetOptions

// newResetOptions returns a struct ready for being used for creating cmd join flags.
func newResetOptions() *resetOptions {
	return &resetOptions{
		certificatesDir: kubeadmapiv1.DefaultCertificatesDir,
		forceReset:      false,
		kubeconfigPath:  kubeadmconstants.GetAdminKubeConfigPath(),
		cleanupTmpDir:   false,
	}
}


// newResetData returns a new resetData struct to be used for the execution of the kubeadm reset workflow.
func newResetData(cmd *cobra.Command, options *resetOptions, in io.Reader, out io.Writer) (*resetData, error) {
	var cfg *kubeadmapi.InitConfiguration

	client, err := cmdutil.GetClientSet(options.kubeconfigPath, false)
	if err == nil {
		klog.V(1).Infof("[reset] Loaded client set from kubeconfig file: %s", options.kubeconfigPath)
		cfg, err = configutil.FetchInitConfigurationFromCluster(client, nil, "reset", false, false)
		if err != nil {
			klog.Warningf("[reset] Unable to fetch the kubeadm-config ConfigMap from cluster: %v", err)
		}
	} else {
		klog.V(1).Infof("[reset] Could not obtain a client set from the kubeconfig file: %s", options.kubeconfigPath)
	}

	ignorePreflightErrorsFromCfg := []string{}
	ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors, ignorePreflightErrorsFromCfg)
	if err != nil {
		return nil, err
	}
	if cfg != nil {
		// Also set the union of pre-flight errors to InitConfiguration, to provide a consistent view of the runtime configuration:
		cfg.NodeRegistration.IgnorePreflightErrors = sets.List(ignorePreflightErrorsSet)
	}

	var criSocketPath string
	if options.criSocketPath == "" {
		criSocketPath, err = resetDetectCRISocket(cfg)
		if err != nil {
			return nil, err
		}
		klog.V(1).Infof("[reset] Detected and using CRI socket: %s", criSocketPath)
	} else {
		criSocketPath = options.criSocketPath
		klog.V(1).Infof("[reset] Using specified CRI socket: %s", criSocketPath)
	}

	return &resetData{
		certificatesDir:       options.certificatesDir,
		client:                client,
		criSocketPath:         criSocketPath,
		forceReset:            options.forceReset,
		ignorePreflightErrors: ignorePreflightErrorsSet,
		inputReader:           in,
		outputWriter:          out,
		cfg:                   cfg,
		dryRun:                options.dryRun,
		cleanupTmpDir:         options.cleanupTmpDir,
	}, nil
}


// AddResetFlags adds reset flags
func AddResetFlags(flagSet *flag.FlagSet, resetOptions *resetOptions) {
	flagSet.StringVar(
		&resetOptions.certificatesDir, options.CertificatesDir, resetOptions.certificatesDir,
		`The path to the directory where the certificates are stored. If specified, clean this directory.`,
	)
	flagSet.BoolVarP(
		&resetOptions.forceReset, options.ForceReset, "f", false,
		"Reset the node without prompting for confirmation.",
	)
	flagSet.BoolVar(
		&resetOptions.dryRun, options.DryRun, resetOptions.dryRun,
		"Don't apply any changes; just output what would be done.",
	)
	flagSet.BoolVar(
		&resetOptions.cleanupTmpDir, options.CleanupTmpDir, resetOptions.cleanupTmpDir,
		fmt.Sprintf("Cleanup the %q directory", path.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.TempDirForKubeadm)),
	)

	options.AddKubeConfigFlag(flagSet, &resetOptions.kubeconfigPath)
	options.AddIgnorePreflightErrorsFlag(flagSet, &resetOptions.ignorePreflightErrors)
	cmdutil.AddCRISocketFlag(flagSet, &resetOptions.criSocketPath)
}


// newCmdReset returns the "kubeadm reset" command
func newCmdReset(in io.Reader, out io.Writer, resetOptions *resetOptions) *cobra.Command {
	if resetOptions == nil {
		resetOptions = newResetOptions()
	}
	resetRunner := workflow.NewRunner()

	cmd := &cobra.Command{
		Use:   "reset",
		Short: "Performs a best effort revert of changes made to this host by 'kubeadm init' or 'kubeadm join'",
		RunE: func(cmd *cobra.Command, args []string) error {
			err := resetRunner.Run(args)
			if err != nil {
				return err
			}

			// output help text instructing user how to remove cni folders
			fmt.Print(cniCleanupInstructions)
			// Output help text instructing user how to remove iptables rules
			fmt.Print(iptablesCleanupInstructions)
			return nil
		},
	}

	AddResetFlags(cmd.Flags(), resetOptions)

	// initialize the workflow runner with the list of phases
	resetRunner.AppendPhase(phases.NewPreflightPhase())
	resetRunner.AppendPhase(phases.NewRemoveETCDMemberPhase())
	resetRunner.AppendPhase(phases.NewCleanupNodePhase())

	// sets the data builder function, that will be used by the runner
	// both when running the entire workflow or single phases
	resetRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) {
		if cmd.Flags().Lookup(options.NodeCRISocket) == nil {
			// avoid CRI detection
			// assume that the command execution does not depend on CRISocket when --cri-socket flag is not set
			resetOptions.criSocketPath = kubeadmconstants.UnknownCRISocket
		}
		return newResetData(cmd, resetOptions, in, out)
	})

	// binds the Runner to kubeadm reset command by altering
	// command help, adding --skip-phases flag and by adding phases subcommands
	resetRunner.BindToCommand(cmd)

	return cmd
}


// Cfg returns the InitConfiguration.
func (r *resetData) Cfg() *kubeadmapi.InitConfiguration {
	return r.cfg
}


func (r *resetData) DryRun() bool

// DryRun returns the dryRun flag.
func (r *resetData) DryRun() bool {
	return r.dryRun
}


// CleanupTmpDir returns the cleanupTmpDir flag.
func (r *resetData) CleanupTmpDir() bool {
	return r.cleanupTmpDir
}


// CertificatesDir returns the CertificatesDir.
func (r *resetData) CertificatesDir() string {
	return r.certificatesDir
}


// Client returns the Client for accessing the cluster.
func (r *resetData) Client() clientset.Interface {
	return r.client
}


func (r *resetData) ForceReset() bool

// ForceReset returns the forceReset flag.
func (r *resetData) ForceReset() bool {
	return r.forceReset
}


// InputReader returns the io.reader used to read messages.
func (r *resetData) InputReader() io.Reader {
	return r.inputReader
}


// IgnorePreflightErrors returns the list of preflight errors to ignore.
func (r *resetData) IgnorePreflightErrors() sets.Set[string] {
	return r.ignorePreflightErrors
}


// CRISocketPath returns the criSocketPath.
func (r *resetData) CRISocketPath() string {
	return r.criSocketPath
}


func resetDetectCRISocket(cfg *kubeadmapi.InitConfiguration) (string, error) {
	if cfg != nil {
		// first try to get the CRI socket from the cluster configuration
		return cfg.NodeRegistration.CRISocket, nil
	}

	// if this fails, try to detect it
	return utilruntime.DetectCRISocket()
}


// newCmdToken returns cobra.Command for token management
func newCmdToken(out io.Writer, errW io.Writer) *cobra.Command {
	var kubeConfigFile string
	var dryRun bool
	tokenCmd := &cobra.Command{
		Use:   "token",
		Short: "Manage bootstrap tokens",
		Long: dedent.Dedent(`
			This command manages bootstrap tokens. It is optional and needed only for advanced use cases.

			In short, bootstrap tokens are used for establishing bidirectional trust between a client and a server.
			A bootstrap token can be used when a client (for example a node that is about to join the cluster) needs
			to trust the server it is talking to. Then a bootstrap token with the "signing" usage can be used.
			bootstrap tokens can also function as a way to allow short-lived authentication to the API Server
			(the token serves as a way for the API Server to trust the client), for example for doing the TLS Bootstrap.

			What is a bootstrap token more exactly?
			 - It is a Secret in the kube-system namespace of type "bootstrap.kubernetes.io/token".
			 - A bootstrap token must be of the form "[a-z0-9]{6}.[a-z0-9]{16}". The former part is the public token ID,
			   while the latter is the Token Secret and it must be kept private at all circumstances!
			 - The name of the Secret must be named "bootstrap-token-(token-id)".

			You can read more about bootstrap tokens here:
			  https://kubernetes.io/docs/admin/bootstrap-tokens/
		`),

		// Without this callback, if a user runs just the "token"
		// command without a subcommand, or with an invalid subcommand,
		// cobra will print usage information, but still exit cleanly.
		// We want to return an error code in these cases so that the
		// user knows that their command was invalid.
		Run: cmdutil.SubCmdRun(),
	}

	options.AddKubeConfigFlag(tokenCmd.PersistentFlags(), &kubeConfigFile)
	tokenCmd.PersistentFlags().BoolVar(&dryRun,
		options.DryRun, dryRun, "Whether to enable dry-run mode or not")

	cfg := cmdutil.DefaultInitConfiguration()

	// Default values for the cobra help text
	kubeadmscheme.Scheme.Default(cfg)

	var cfgPath string
	var printJoinCommand bool
	var certificateKey string
	bto := options.NewBootstrapTokenOptions()

	createCmd := &cobra.Command{
		Use:                   "create [token]",
		DisableFlagsInUseLine: true,
		Short:                 "Create bootstrap tokens on the server",
		Long: dedent.Dedent(`
			This command will create a bootstrap token for you.
			You can specify the usages for this token, the "time to live" and an optional human friendly description.

			The [token] is the actual token to write.
			This should be a securely generated random token of the form "[a-z0-9]{6}.[a-z0-9]{16}".
			If no [token] is given, kubeadm will generate a random token instead.
		`),
		RunE: func(tokenCmd *cobra.Command, args []string) error {
			if len(args) > 0 {
				bto.TokenStr = args[0]
			}
			klog.V(1).Infoln("[token] validating mixed arguments")
			if err := validation.ValidateMixedArguments(tokenCmd.Flags()); err != nil {
				return err
			}

			if err := bto.ApplyTo(cfg); err != nil {
				return err
			}

			klog.V(1).Infoln("[token] getting Clientsets from kubeconfig file")
			kubeConfigFile = cmdutil.GetKubeConfigPath(kubeConfigFile)
			client, err := cmdutil.GetClientSet(kubeConfigFile, dryRun)
			if err != nil {
				return err
			}

			return RunCreateToken(out, client, cfgPath, cfg, printJoinCommand, certificateKey, kubeConfigFile)
		},
	}

	options.AddConfigFlag(createCmd.Flags(), &cfgPath)
	createCmd.Flags().BoolVar(&printJoinCommand,
		"print-join-command", false, "Instead of printing only the token, print the full 'kubeadm join' flag needed to join the cluster using the token.")
	createCmd.Flags().StringVar(&certificateKey,
		options.CertificateKey, "", "When used together with '--print-join-command', print the full 'kubeadm join' flag needed to join the cluster as a control-plane. To create a new certificate key you must use 'kubeadm init phase upload-certs --upload-certs'.")
	bto.AddTTLFlagWithName(createCmd.Flags(), "ttl")
	bto.AddUsagesFlag(createCmd.Flags())
	bto.AddGroupsFlag(createCmd.Flags())
	bto.AddDescriptionFlag(createCmd.Flags())

	tokenCmd.AddCommand(createCmd)
	tokenCmd.AddCommand(newCmdTokenGenerate(out))

	outputFlags := output.NewOutputFlags(&tokenTextPrintFlags{}).WithTypeSetter(outputapischeme.Scheme).WithDefaultOutput(output.TextOutput)

	listCmd := &cobra.Command{
		Use:   "list",
		Short: "List bootstrap tokens on the server",
		Long: dedent.Dedent(`
			This command will list all bootstrap tokens for you.
		`),
		RunE: func(tokenCmd *cobra.Command, args []string) error {
			kubeConfigFile = cmdutil.GetKubeConfigPath(kubeConfigFile)
			client, err := cmdutil.GetClientSet(kubeConfigFile, dryRun)
			if err != nil {
				return err
			}

			printer, err := outputFlags.ToPrinter()
			if err != nil {
				return errors.Wrap(err, "could not construct output printer")
			}

			return RunListTokens(out, errW, client, printer)
		},
		Args: cobra.NoArgs,
	}

	outputFlags.AddFlags(listCmd)

	tokenCmd.AddCommand(listCmd)

	deleteCmd := &cobra.Command{
		Use:                   "delete [token-value] ...",
		DisableFlagsInUseLine: true,
		Short:                 "Delete bootstrap tokens on the server",
		Long: dedent.Dedent(`
			This command will delete a list of bootstrap tokens for you.

			The [token-value] is the full Token of the form "[a-z0-9]{6}.[a-z0-9]{16}" or the
			Token ID of the form "[a-z0-9]{6}" to delete.
		`),
		RunE: func(tokenCmd *cobra.Command, args []string) error {
			if len(args) < 1 {
				return errors.Errorf("missing argument; 'token delete' is missing token of form %q or %q", bootstrapapi.BootstrapTokenPattern, bootstrapapi.BootstrapTokenIDPattern)
			}
			kubeConfigFile = cmdutil.GetKubeConfigPath(kubeConfigFile)
			client, err := cmdutil.GetClientSet(kubeConfigFile, dryRun)
			if err != nil {
				return err
			}

			return RunDeleteTokens(out, client, args)
		},
	}
	tokenCmd.AddCommand(deleteCmd)

	return tokenCmd
}


// newCmdTokenGenerate returns cobra.Command to generate new token
func newCmdTokenGenerate(out io.Writer) *cobra.Command {
	return &cobra.Command{
		Use:   "generate",
		Short: "Generate and print a bootstrap token, but do not create it on the server",
		Long: dedent.Dedent(`
			This command will print out a randomly-generated bootstrap token that can be used with
			the "init" and "join" commands.

			You don't have to use this command in order to generate a token. You can do so
			yourself as long as it is in the format "[a-z0-9]{6}.[a-z0-9]{16}". This
			command is provided for convenience to generate tokens in the given format.

			You can also use "kubeadm init" without specifying a token and it will
			generate and print one for you.
		`),
		RunE: func(cmd *cobra.Command, args []string) error {
			return RunGenerateToken(out)
		},
		Args: cobra.NoArgs,
	}
}


// RunCreateToken generates a new bootstrap token and stores it as a secret on the server.
func RunCreateToken(out io.Writer, client clientset.Interface, cfgPath string, initCfg *kubeadmapiv1.InitConfiguration, printJoinCommand bool, certificateKey string, kubeConfigFile string) error {
	// ClusterConfiguration is needed just for the call to LoadOrDefaultInitConfiguration
	clusterCfg := &kubeadmapiv1.ClusterConfiguration{
		// KubernetesVersion is not used, but we set this explicitly to avoid
		// the lookup of the version from the internet when executing LoadOrDefaultInitConfiguration
		KubernetesVersion: kubeadmconstants.CurrentKubernetesVersion.String(),
	}
	kubeadmscheme.Scheme.Default(clusterCfg)

	// This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags
	klog.V(1).Infoln("[token] loading configurations")

	internalcfg, err := configutil.LoadOrDefaultInitConfiguration(cfgPath, initCfg, clusterCfg)
	if err != nil {
		return err
	}

	klog.V(1).Infoln("[token] creating token")
	if err := tokenphase.CreateNewTokens(client, internalcfg.BootstrapTokens); err != nil {
		return err
	}

	// if --print-join-command was specified, print a machine-readable full `kubeadm join` command
	// otherwise, just print the token
	if printJoinCommand {
		skipTokenPrint := false
		if certificateKey != "" {
			skipCertificateKeyPrint := false
			joinCommand, err := cmdutil.GetJoinControlPlaneCommand(kubeConfigFile, internalcfg.BootstrapTokens[0].Token.String(), certificateKey, skipTokenPrint, skipCertificateKeyPrint)
			if err != nil {
				return errors.Wrap(err, "failed to get join command")
			}
			joinCommand = strings.ReplaceAll(joinCommand, "\\\n", "")
			joinCommand = strings.ReplaceAll(joinCommand, "\t", "")
			fmt.Fprintln(out, joinCommand)
		} else {
			joinCommand, err := cmdutil.GetJoinWorkerCommand(kubeConfigFile, internalcfg.BootstrapTokens[0].Token.String(), skipTokenPrint)
			if err != nil {
				return errors.Wrap(err, "failed to get join command")
			}
			joinCommand = strings.ReplaceAll(joinCommand, "\\\n", "")
			joinCommand = strings.ReplaceAll(joinCommand, "\t", "")
			fmt.Fprintln(out, joinCommand)
		}
	} else {
		if certificateKey != "" {
			return errors.New("cannot use --certificate-key without --print-join-command")
		}
		fmt.Fprintln(out, internalcfg.BootstrapTokens[0].Token.String())
	}

	return nil
}


// RunGenerateToken just generates a random token for the user
func RunGenerateToken(out io.Writer) error {
	klog.V(1).Infoln("[token] generating random token")
	token, err := bootstraputil.GenerateBootstrapToken()
	if err != nil {
		return err
	}

	fmt.Fprintln(out, token)
	return nil
}


func formatBootstrapToken(obj *outputapiv1alpha2.BootstrapToken) string {
	ttl := "<forever>"
	expires := "<never>"
	if obj.Expires != nil {
		ttl = duration.ShortHumanDuration(time.Until(obj.Expires.Time))
		expires = obj.Expires.Format(time.RFC3339)
	}
	ttl = fmt.Sprintf("%-9s", ttl)

	usages := strings.Join(obj.Usages, ",")
	if len(usages) == 0 {
		usages = "<none>"
	}
	usages = fmt.Sprintf("%-22s", usages)

	description := obj.Description
	if len(description) == 0 {
		description = "<none>"
	}
	description = fmt.Sprintf("%-56s", description)

	groups := strings.Join(obj.Groups, ",")
	if len(groups) == 0 {
		groups = "<none>"
	}

	return fmt.Sprintf("%s\t%s\t%s\t%s\t%s\t%s\n", obj.Token, ttl, expires, usages, description, groups)
}


// PrintObj is an implementation of ResourcePrinter.PrintObj for plain text output
func (ttp *tokenTextPrinter) PrintObj(obj runtime.Object, writer io.Writer) error {
	tabw := tabwriter.NewWriter(writer, 10, 4, 3, ' ', 0)

	// Print header
	if !ttp.headerIsPrinted {
		fmt.Fprintln(tabw, strings.Join(ttp.columns, "\t"))
		ttp.headerIsPrinted = true
	}

	// Print token
	fmt.Fprint(tabw, formatBootstrapToken(obj.(*outputapiv1alpha2.BootstrapToken)))

	return tabw.Flush()
}


// ToPrinter returns a kubeadm printer for the text output format
func (tpf *tokenTextPrintFlags) ToPrinter(outputFormat string) (output.Printer, error) {
	if outputFormat == output.TextOutput {
		return &tokenTextPrinter{columns: []string{"TOKEN", "TTL", "EXPIRES", "USAGES", "DESCRIPTION", "EXTRA GROUPS"}}, nil
	}
	return nil, genericclioptions.NoCompatiblePrinterError{OutputFormat: &outputFormat, AllowedFormats: []string{output.TextOutput}}
}


// RunListTokens lists details on all existing bootstrap tokens on the server.
func RunListTokens(out io.Writer, errW io.Writer, client clientset.Interface, printer output.Printer) error {
	// First, build our selector for bootstrap tokens only
	klog.V(1).Infoln("[token] preparing selector for bootstrap token")
	tokenSelector := fields.SelectorFromSet(
		map[string]string{
			"type": string(bootstrapapi.SecretTypeBootstrapToken),
		},
	)
	listOptions := metav1.ListOptions{
		FieldSelector: tokenSelector.String(),
	}

	klog.V(1).Info("[token] retrieving list of bootstrap tokens")
	secrets, err := client.CoreV1().Secrets(metav1.NamespaceSystem).List(context.TODO(), listOptions)
	if err != nil {
		return errors.Wrap(err, "failed to list bootstrap tokens")
	}

	for _, secret := range secrets.Items {
		// Get the BootstrapToken struct representation from the Secret object
		token, err := bootstraptokenv1.BootstrapTokenFromSecret(&secret)
		if err != nil {
			fmt.Fprintf(errW, "%v", err)
			continue
		}

		// Convert token into versioned output structure
		outputToken := outputapiv1alpha2.BootstrapToken{
			BootstrapToken: bootstraptokenv1.BootstrapToken{
				Token:       &bootstraptokenv1.BootstrapTokenString{ID: token.Token.ID, Secret: token.Token.Secret},
				Description: token.Description,
				TTL:         token.TTL,
				Expires:     token.Expires,
				Usages:      token.Usages,
				Groups:      token.Groups,
			},
		}

		if err := printer.PrintObj(&outputToken, out); err != nil {
			return errors.Wrapf(err, "unable to print token %s", token.Token)
		}
	}
	return nil
}


// RunDeleteTokens removes a bootstrap tokens from the server.
func RunDeleteTokens(out io.Writer, client clientset.Interface, tokenIDsOrTokens []string) error {
	for _, tokenIDOrToken := range tokenIDsOrTokens {
		// Assume this is a token id and try to parse it
		tokenID := tokenIDOrToken
		klog.V(1).Info("[token] parsing token")
		if !bootstraputil.IsValidBootstrapTokenID(tokenIDOrToken) {
			// Okay, the full token with both id and secret was probably passed. Parse it and extract the ID only
			bts, err := bootstraptokenv1.NewBootstrapTokenString(tokenIDOrToken)
			if err != nil {
				return errors.Errorf("given token didn't match pattern %q or %q",
					bootstrapapi.BootstrapTokenPattern, bootstrapapi.BootstrapTokenIDPattern)
			}
			tokenID = bts.ID
		}

		tokenSecretName := bootstraputil.BootstrapTokenSecretName(tokenID)
		klog.V(1).Infof("[token] deleting token %q", tokenID)
		if err := client.CoreV1().Secrets(metav1.NamespaceSystem).Delete(context.TODO(), tokenSecretName, metav1.DeleteOptions{}); err != nil {
			return errors.Wrapf(err, "failed to delete bootstrap token %q", tokenID)
		}
		fmt.Fprintf(out, "bootstrap token %q deleted\n", tokenID)
	}
	return nil
}


// sessionIsInteractive returns true if the session is of an interactive type (the default, can be opted out of with -y, -f or --dry-run)
func (f *applyFlags) sessionIsInteractive() bool {
	return !(f.nonInteractiveMode || f.dryRun || f.force)
}


// newCmdApply returns the cobra command for `kubeadm upgrade apply`
func newCmdApply(apf *applyPlanFlags) *cobra.Command {
	flags := &applyFlags{
		applyPlanFlags: apf,
		etcdUpgrade:    true,
		renewCerts:     true,
	}

	cmd := &cobra.Command{
		Use:                   "apply [version]",
		DisableFlagsInUseLine: true,
		Short:                 "Upgrade your Kubernetes cluster to the specified version",
		RunE: func(cmd *cobra.Command, args []string) error {
			return runApply(flags, args)
		},
	}

	// Register the common flags for apply and plan
	addApplyPlanFlags(cmd.Flags(), flags.applyPlanFlags)
	// Specify the valid flags specific for apply
	cmd.Flags().BoolVarP(&flags.nonInteractiveMode, "yes", "y", flags.nonInteractiveMode, "Perform the upgrade and do not prompt for confirmation (non-interactive mode).")
	cmd.Flags().BoolVarP(&flags.force, "force", "f", flags.force, "Force upgrading although some requirements might not be met. This also implies non-interactive mode.")
	cmd.Flags().BoolVar(&flags.dryRun, options.DryRun, flags.dryRun, "Do not change any state, just output what actions would be performed.")
	cmd.Flags().BoolVar(&flags.etcdUpgrade, "etcd-upgrade", flags.etcdUpgrade, "Perform the upgrade of etcd.")
	cmd.Flags().BoolVar(&flags.renewCerts, options.CertificateRenewal, flags.renewCerts, "Perform the renewal of certificates used by component changed during upgrades.")
	options.AddPatchesFlag(cmd.Flags(), &flags.patchesDir)

	return cmd
}


// runApply takes care of the actual upgrade functionality
// It does the following things:
// - Checks if the cluster is healthy
// - Gets the configuration from the kubeadm-config ConfigMap in the cluster
// - Enforces all version skew policies
// - Asks the user if they really want to upgrade
// - Makes sure the control plane images are available locally on the control-plane(s)
// - Upgrades the control plane components
// - Applies the other resources that'd be created with kubeadm init as well, like
//   - Creating the RBAC rules for the bootstrap tokens and the cluster-info ConfigMap
//   - Applying new CorDNS and kube-proxy manifests
//   - Uploads the newly used configuration to the cluster ConfigMap
func runApply(flags *applyFlags, args []string) error {

	// Start with the basics, verify that the cluster is healthy and get the configuration from the cluster (using the ConfigMap)
	klog.V(1).Infoln("[upgrade/apply] verifying health of cluster")
	klog.V(1).Infoln("[upgrade/apply] retrieving configuration from cluster")
	client, versionGetter, cfg, err := enforceRequirements(flags.applyPlanFlags, args, flags.dryRun, true, &output.TextPrinter{}, loadConfig)
	if err != nil {
		return err
	}

	// Validate requested and validate actual version
	klog.V(1).Infoln("[upgrade/apply] validating requested and actual version")
	if err := configutil.NormalizeKubernetesVersion(&cfg.ClusterConfiguration); err != nil {
		return err
	}

	// Use normalized version string in all following code.
	newK8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
	if err != nil {
		return errors.Errorf("unable to parse normalized version %q as a semantic version", cfg.KubernetesVersion)
	}

	if err := features.ValidateVersion(features.InitFeatureGates, cfg.FeatureGates, cfg.KubernetesVersion); err != nil {
		return err
	}

	// Enforce the version skew policies
	klog.V(1).Infoln("[upgrade/version] enforcing version skew policies")
	if err := EnforceVersionPolicies(cfg.KubernetesVersion, newK8sVersion, flags, versionGetter); err != nil {
		return errors.Wrap(err, "[upgrade/version] FATAL")
	}

	// If the current session is interactive, ask the user whether they really want to upgrade.
	if flags.sessionIsInteractive() {
		if err := cmdutil.InteractivelyConfirmAction("upgrade", "Are you sure you want to proceed?", os.Stdin); err != nil {
			return err
		}
	}

	if !flags.dryRun {
		fmt.Println("[upgrade/prepull] Pulling images required for setting up a Kubernetes cluster")
		fmt.Println("[upgrade/prepull] This might take a minute or two, depending on the speed of your internet connection")
		fmt.Println("[upgrade/prepull] You can also perform this action in beforehand using 'kubeadm config images pull'")
		if err := preflight.RunPullImagesCheck(utilsexec.New(), cfg, sets.New(cfg.NodeRegistration.IgnorePreflightErrors...)); err != nil {
			return err
		}
	} else {
		fmt.Println("[upgrade/prepull] Would pull the required images (like 'kubeadm config images pull')")
	}

	waiter := getWaiter(flags.dryRun, client, upgrade.UpgradeManifestTimeout)

	// Now; perform the upgrade procedure
	if err := PerformControlPlaneUpgrade(flags, client, waiter, cfg); err != nil {
		return errors.Wrap(err, "[upgrade/apply] FATAL")
	}

	// Upgrade RBAC rules and addons.
	klog.V(1).Infoln("[upgrade/postupgrade] upgrading RBAC rules and addons")
	if err := upgrade.PerformPostUpgradeTasks(client, cfg, flags.patchesDir, flags.dryRun, flags.applyPlanFlags.out); err != nil {
		return errors.Wrap(err, "[upgrade/postupgrade] FATAL post-upgrade error")
	}

	if flags.dryRun {
		fmt.Println("[upgrade/successful] Finished dryrunning successfully!")
		return nil
	}

	fmt.Println("")
	fmt.Printf("[upgrade/successful] SUCCESS! Your cluster was upgraded to %q. Enjoy!\n", cfg.KubernetesVersion)
	fmt.Println("")
	fmt.Println("[upgrade/kubelet] Now that your control plane is upgraded, please proceed with upgrading your kubelets if you haven't already done so.")

	return nil
}


// EnforceVersionPolicies makes sure that the version the user specified is valid to upgrade to
// There are both fatal and skippable (with --force) errors
func EnforceVersionPolicies(newK8sVersionStr string, newK8sVersion *version.Version, flags *applyFlags, versionGetter upgrade.VersionGetter) error {
	fmt.Printf("[upgrade/version] You have chosen to change the cluster version to %q\n", newK8sVersionStr)

	versionSkewErrs := upgrade.EnforceVersionPolicies(versionGetter, newK8sVersionStr, newK8sVersion, flags.allowExperimentalUpgrades, flags.allowRCUpgrades)
	if versionSkewErrs != nil {

		if len(versionSkewErrs.Mandatory) > 0 {
			return errors.Errorf("the --version argument is invalid due to these fatal errors:\n\n%v\nPlease fix the misalignments highlighted above and try upgrading again",
				kubeadmutil.FormatErrMsg(versionSkewErrs.Mandatory))
		}

		if len(versionSkewErrs.Skippable) > 0 {
			// Return the error if the user hasn't specified the --force flag
			if !flags.force {
				return errors.Errorf("the --version argument is invalid due to these errors:\n\n%v\nCan be bypassed if you pass the --force flag",
					kubeadmutil.FormatErrMsg(versionSkewErrs.Skippable))
			}
			// Soft errors found, but --force was specified
			fmt.Printf("[upgrade/version] Found %d potential version compatibility errors but skipping since the --force flag is set: \n\n%v", len(versionSkewErrs.Skippable), kubeadmutil.FormatErrMsg(versionSkewErrs.Skippable))
		}
	}
	return nil
}


// PerformControlPlaneUpgrade actually performs the upgrade procedure for the cluster of your type (self-hosted or static-pod-hosted)
func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.InitConfiguration) error {

	// OK, the cluster is hosted using static pods. Upgrade a static-pod hosted cluster
	fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q (timeout: %v)...\n",
		internalcfg.KubernetesVersion, upgrade.UpgradeManifestTimeout)

	if flags.dryRun {
		return upgrade.DryRunStaticPodUpgrade(flags.patchesDir, internalcfg)
	}

	return upgrade.PerformStaticPodUpgrade(client, waiter, internalcfg, flags.etcdUpgrade, flags.renewCerts, flags.patchesDir)
}


// isKubeadmConfigPresent checks if a kubeadm config type is found in the provided document map
func isKubeadmConfigPresent(docmap kubeadmapi.DocumentMap) bool {
	for gvk := range docmap {
		if gvk.Group == kubeadmapi.GroupName {
			return true
		}
	}
	return false
}


// loadConfig loads configuration from a file and/or the cluster. InitConfiguration, ClusterConfiguration and (optionally) component configs
// are loaded. This function allows the component configs to be loaded from a file that contains only them. If the file contains any kubeadm types
// in it (API group "kubeadm.kubernetes.io" present), then the supplied file is treaded as a legacy reconfiguration style "--config" use and the
// returned bool value is set to true (the only case to be done so).
func loadConfig(cfgPath string, client clientset.Interface, skipComponentConfigs bool, printer output.Printer) (*kubeadmapi.InitConfiguration, bool, error) {
	// Used for info logs here
	const logPrefix = "upgrade/config"

	// The usual case here is to not have a config file, but rather load the config from the cluster.
	// This is probably 90% of the time. So we handle it first.
	if cfgPath == "" {
		cfg, err := configutil.FetchInitConfigurationFromCluster(client, printer, logPrefix, false, skipComponentConfigs)
		return cfg, false, err
	}

	// Otherwise, we have a config file. Let's load it.
	configBytes, err := os.ReadFile(cfgPath)
	if err != nil {
		return nil, false, errors.Wrapf(err, "unable to load config from file %q", cfgPath)
	}

	// Split the YAML documents in the file into a DocumentMap
	docmap, err := kubeadmutil.SplitYAMLDocuments(configBytes)
	if err != nil {
		return nil, false, err
	}

	// If there are kubeadm types (API group kubeadm.kubernetes.io) present, we need to keep the existing behavior
	// here. Basically, we have to load all of the configs from the file and none from the cluster. Configs that are
	// missing from the file will be automatically regenerated by kubeadm even if they are present in the cluster.
	// The resulting configs overwrite the existing cluster ones at the end of a successful upgrade apply operation.
	if isKubeadmConfigPresent(docmap) {
		klog.Warning("WARNING: Usage of the --config flag with kubeadm config types for reconfiguring the cluster during upgrade is not recommended!")
		cfg, err := configutil.BytesToInitConfiguration(configBytes)
		return cfg, true, err
	}

	// If no kubeadm config types are present, we assume that there are manually upgraded component configs in the file.
	// Hence, we load the kubeadm types from the cluster.
	initCfg, err := configutil.FetchInitConfigurationFromCluster(client, printer, logPrefix, false, true)
	if err != nil {
		return nil, false, err
	}

	// Stop here if the caller does not want us to load the component configs
	if !skipComponentConfigs {
		// Load the component configs with upgrades
		if err := componentconfigs.FetchFromClusterWithLocalOverwrites(&initCfg.ClusterConfiguration, client, docmap); err != nil {
			return nil, false, err
		}

		// Now default and validate the configs
		componentconfigs.Default(&initCfg.ClusterConfiguration, &initCfg.LocalAPIEndpoint, &initCfg.NodeRegistration)
		if errs := componentconfigs.Validate(&initCfg.ClusterConfiguration); len(errs) != 0 {
			return nil, false, errs.ToAggregate()
		}
	}

	return initCfg, false, nil
}


// enforceRequirements verifies that it's okay to upgrade and then returns the variables needed for the rest of the procedure
func enforceRequirements(flags *applyPlanFlags, args []string, dryRun bool, upgradeApply bool, printer output.Printer, loadConfig LoadConfigFunc) (clientset.Interface, upgrade.VersionGetter, *kubeadmapi.InitConfiguration, error) {
	client, err := getClient(flags.kubeConfigPath, dryRun)
	if err != nil {
		return nil, nil, nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", flags.kubeConfigPath)
	}

	// Fetch the configuration from a file or ConfigMap and validate it
	printer.Printf("[upgrade/config] Making sure the configuration is correct:\n")

	var newK8sVersion string
	cfg, legacyReconfigure, err := loadConfig(flags.cfgPath, client, !upgradeApply, printer)
	if err != nil {
		if apierrors.IsNotFound(err) {
			printer.Printf("[upgrade/config] In order to upgrade, a ConfigMap called %q in the %s namespace must exist.\n", constants.KubeadmConfigConfigMap, metav1.NamespaceSystem)
			printer.Printf("[upgrade/config] Without this information, 'kubeadm upgrade' won't know how to configure your upgraded cluster.\n")
			printer.Println()
			printer.Printf("[upgrade/config] Next steps:\n")
			printer.Printf("\t- OPTION 1: Run 'kubeadm config upload from-flags' and specify the same CLI arguments you passed to 'kubeadm init' when you created your control-plane.\n")
			printer.Printf("\t- OPTION 2: Run 'kubeadm config upload from-file' and specify the same config file you passed to 'kubeadm init' when you created your control-plane.\n")
			printer.Printf("\t- OPTION 3: Pass a config file to 'kubeadm upgrade' using the --config flag.\n")
			printer.Println()
			err = errors.Errorf("the ConfigMap %q in the %s namespace used for getting configuration information was not found", constants.KubeadmConfigConfigMap, metav1.NamespaceSystem)
		}
		return nil, nil, nil, errors.Wrap(err, "[upgrade/config] FATAL")
	} else if legacyReconfigure {
		// Set the newK8sVersion to the value in the ClusterConfiguration. This is done, so that users who use the --config option
		// to supply a new ClusterConfiguration don't have to specify the Kubernetes version twice,
		// if they don't want to upgrade but just change a setting.
		newK8sVersion = cfg.KubernetesVersion
	}

	// The version arg is mandatory, during upgrade apply, unless it's specified in the config file
	if upgradeApply && newK8sVersion == "" {
		if err := cmdutil.ValidateExactArgNumber(args, []string{"version"}); err != nil {
			return nil, nil, nil, err
		}
	}

	// If option was specified in both args and config file, args will overwrite the config file.
	if len(args) == 1 {
		newK8sVersion = args[0]
		if upgradeApply {
			// The `upgrade apply` version always overwrites the KubernetesVersion in the returned cfg with the target
			// version. While this is not the same for `upgrade plan` where the KubernetesVersion should be the old
			// one (because the call to getComponentConfigVersionStates requires the currently installed version).
			// This also makes the KubernetesVersion value returned for `upgrade plan` consistent as that command
			// allows to not specify a target version in which case KubernetesVersion will always hold the currently
			// installed one.
			cfg.KubernetesVersion = newK8sVersion
		}
	}

	ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(flags.ignorePreflightErrors, cfg.NodeRegistration.IgnorePreflightErrors)
	if err != nil {
		return nil, nil, nil, err
	}
	// Also set the union of pre-flight errors to InitConfiguration, to provide a consistent view of the runtime configuration:
	cfg.NodeRegistration.IgnorePreflightErrors = sets.List(ignorePreflightErrorsSet)

	// Ensure the user is root
	klog.V(1).Info("running preflight checks")
	if err := runPreflightChecks(client, ignorePreflightErrorsSet, &cfg.ClusterConfiguration, printer); err != nil {
		return nil, nil, nil, err
	}

	// Run healthchecks against the cluster
	if err := upgrade.CheckClusterHealth(client, &cfg.ClusterConfiguration, ignorePreflightErrorsSet); err != nil {
		return nil, nil, nil, errors.Wrap(err, "[upgrade/health] FATAL")
	}

	// If features gates are passed to the command line, use it (otherwise use featureGates from configuration)
	if flags.featureGatesString != "" {
		cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, flags.featureGatesString)
		if err != nil {
			return nil, nil, nil, errors.Wrap(err, "[upgrade/config] FATAL")
		}
	}

	// Check if feature gate flags used in the cluster are consistent with the set of features currently supported by kubeadm
	if msg := features.CheckDeprecatedFlags(&features.InitFeatureGates, cfg.FeatureGates); len(msg) > 0 {
		for _, m := range msg {
			printer.Printf("[upgrade/config] %s\n", m)
		}
		return nil, nil, nil, errors.New("[upgrade/config] FATAL. Unable to upgrade a cluster using deprecated feature-gate flags. Please see the release notes")
	}

	// If the user told us to print this information out; do it!
	if flags.printConfig {
		printConfiguration(&cfg.ClusterConfiguration, os.Stdout, printer)
	}

	// Use a real version getter interface that queries the API server, the kubeadm client and the Kubernetes CI system for latest versions
	return client, upgrade.NewOfflineVersionGetter(upgrade.NewKubeVersionGetter(client), newK8sVersion), cfg, nil
}


// printConfiguration prints the external version of the API to yaml
func printConfiguration(clustercfg *kubeadmapi.ClusterConfiguration, w io.Writer, printer output.Printer) {
	// Short-circuit if cfg is nil, so we can safely get the value of the pointer below
	if clustercfg == nil {
		return
	}

	cfgYaml, err := configutil.MarshalKubeadmConfigObject(clustercfg)
	if err == nil {
		printer.Fprintln(w, "[upgrade/config] Configuration used:")

		scanner := bufio.NewScanner(bytes.NewReader(cfgYaml))
		for scanner.Scan() {
			printer.Fprintf(w, "\t%s\n", scanner.Text())
		}
	}
}


// runPreflightChecks runs the root preflight check
func runPreflightChecks(client clientset.Interface, ignorePreflightErrors sets.Set[string], cfg *kubeadmapi.ClusterConfiguration, printer output.Printer) error {
	printer.Printf("[preflight] Running pre-flight checks.\n")
	err := preflight.RunRootCheckOnly(ignorePreflightErrors)
	if err != nil {
		return err
	}
	return upgrade.RunCoreDNSMigrationCheck(client, ignorePreflightErrors)
}


// getClient gets a real or fake client depending on whether the user is dry-running or not
func getClient(file string, dryRun bool) (clientset.Interface, error) {
	if dryRun {
		dryRunGetter, err := apiclient.NewClientBackedDryRunGetterFromKubeconfig(file)
		if err != nil {
			return nil, err
		}

		// In order for fakeclient.Discovery().ServerVersion() to return the backing API Server's
		// real version; we have to do some clever API machinery tricks. First, we get the real
		// API Server's version
		realServerVersion, err := dryRunGetter.Client().Discovery().ServerVersion()
		if err != nil {
			return nil, errors.Wrap(err, "failed to get server version")
		}

		// Get the fake clientset
		dryRunOpts := apiclient.GetDefaultDryRunClientOptions(dryRunGetter, os.Stdout)
		// Print GET and LIST requests
		dryRunOpts.PrintGETAndLIST = true
		fakeclient := apiclient.NewDryRunClientWithOpts(dryRunOpts)
		// As we know the return of Discovery() of the fake clientset is of type *fakediscovery.FakeDiscovery
		// we can convert it to that struct.
		fakeclientDiscovery, ok := fakeclient.Discovery().(*fakediscovery.FakeDiscovery)
		if !ok {
			return nil, errors.New("couldn't set fake discovery's server version")
		}
		// Lastly, set the right server version to be used
		fakeclientDiscovery.FakedServerVersion = realServerVersion
		// return the fake clientset used for dry-running
		return fakeclient, nil
	}
	return kubeconfigutil.ClientSetFromFile(file)
}


// getWaiter gets the right waiter implementation
func getWaiter(dryRun bool, client clientset.Interface, timeout time.Duration) apiclient.Waiter {
	if dryRun {
		return dryrunutil.NewWaiter()
	}
	return apiclient.NewKubeWaiter(client, timeout, os.Stdout)
}


// newCmdDiff returns the cobra command for `kubeadm upgrade diff`
func newCmdDiff(out io.Writer) *cobra.Command {
	flags := &diffFlags{
		kubeConfigPath: constants.GetAdminKubeConfigPath(),
		out:            out,
	}

	cmd := &cobra.Command{
		Use:   "diff [version]",
		Short: "Show what differences would be applied to existing static pod manifests. See also: kubeadm upgrade apply --dry-run",
		RunE: func(cmd *cobra.Command, args []string) error {
			// Run preflight checks for diff to check that the manifests already exist.
			if err := validateManifestsPath(
				flags.apiServerManifestPath,
				flags.controllerManagerManifestPath,
				flags.schedulerManifestPath); err != nil {
				return err
			}
			return runDiff(flags, args)
		},
	}

	options.AddKubeConfigFlag(cmd.Flags(), &flags.kubeConfigPath)
	options.AddConfigFlag(cmd.Flags(), &flags.cfgPath)
	cmd.Flags().StringVar(&flags.apiServerManifestPath, "api-server-manifest", defaultAPIServerManifestPath, "path to API server manifest")
	cmd.Flags().StringVar(&flags.controllerManagerManifestPath, "controller-manager-manifest", defaultControllerManagerManifestPath, "path to controller manifest")
	cmd.Flags().StringVar(&flags.schedulerManifestPath, "scheduler-manifest", defaultSchedulerManifestPath, "path to scheduler manifest")
	cmd.Flags().IntVarP(&flags.contextLines, "context-lines", "c", 3, "How many lines of context in the diff")

	return cmd
}


func validateManifestsPath(manifests ...string) (err error) {
	for _, manifestPath := range manifests {
		if len(manifestPath) == 0 {
			return errors.New("empty manifest path")
		}
		s, err := os.Stat(manifestPath)
		if err != nil {
			if os.IsNotExist(err) {
				return errors.Wrapf(err, "the manifest file %q does not exist", manifestPath)
			}
			return errors.Wrapf(err, "error obtaining stats for manifest file %q", manifestPath)
		}
		if s.IsDir() {
			return errors.Errorf("%q is a directory", manifestPath)
		}
	}
	return nil
}


func runDiff(flags *diffFlags, args []string) error {
	var err error
	var cfg *kubeadmapi.InitConfiguration
	if flags.cfgPath != "" {
		cfg, err = configutil.LoadInitConfigurationFromFile(flags.cfgPath)
	} else {
		var client *client.Clientset
		client, err = kubeconfigutil.ClientSetFromFile(flags.kubeConfigPath)
		if err != nil {
			return errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", flags.kubeConfigPath)
		}
		cfg, err = configutil.FetchInitConfigurationFromCluster(client, nil, "upgrade/diff", false, false)
	}
	if err != nil {
		return err
	}

	// If the version is specified in config file, pick up that value.
	if cfg.KubernetesVersion != "" {
		flags.newK8sVersionStr = cfg.KubernetesVersion
	}

	// If the new version is already specified in config file, version arg is optional.
	if flags.newK8sVersionStr == "" {
		if err := cmdutil.ValidateExactArgNumber(args, []string{"version"}); err != nil {
			return err
		}
	}

	// If option was specified in both args and config file, args will overwrite the config file.
	if len(args) == 1 {
		flags.newK8sVersionStr = args[0]
	}

	_, err = version.ParseSemantic(flags.newK8sVersionStr)
	if err != nil {
		return err
	}

	cfg.ClusterConfiguration.KubernetesVersion = flags.newK8sVersionStr

	specs := controlplane.GetStaticPodSpecs(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint)
	for spec, pod := range specs {
		var path string
		switch spec {
		case constants.KubeAPIServer:
			path = flags.apiServerManifestPath
		case constants.KubeControllerManager:
			path = flags.controllerManagerManifestPath
		case constants.KubeScheduler:
			path = flags.schedulerManifestPath
		default:
			klog.Errorf("[diff] unknown spec %v", spec)
			continue
		}

		newManifest, err := kubeadmutil.MarshalToYaml(&pod, corev1.SchemeGroupVersion)
		if err != nil {
			return err
		}
		if path == "" {
			return errors.New("empty manifest path")
		}
		existingManifest, err := os.ReadFile(path)
		if err != nil {
			return err
		}

		// Populated and write out the diff
		diff := difflib.UnifiedDiff{
			A:        difflib.SplitLines(string(existingManifest)),
			B:        difflib.SplitLines(string(newManifest)),
			FromFile: path,
			ToFile:   "new manifest",
			Context:  flags.contextLines,
		}

		difflib.WriteUnifiedDiff(flags.out, diff)
	}
	return nil
}


// newCmdNode returns the cobra command for `kubeadm upgrade node`
func newCmdNode(out io.Writer) *cobra.Command {
	nodeOptions := newNodeOptions()
	nodeRunner := workflow.NewRunner()

	cmd := &cobra.Command{
		Use:   "node",
		Short: "Upgrade commands for a node in the cluster",
		RunE: func(cmd *cobra.Command, args []string) error {
			return nodeRunner.Run(args)
		},
		Args: cobra.NoArgs,
	}

	// adds flags to the node command
	// flags could be eventually inherited by the sub-commands automatically generated for phases
	addUpgradeNodeFlags(cmd.Flags(), nodeOptions)
	options.AddPatchesFlag(cmd.Flags(), &nodeOptions.patchesDir)

	// initialize the workflow runner with the list of phases
	nodeRunner.AppendPhase(phases.NewPreflightPhase())
	nodeRunner.AppendPhase(phases.NewControlPlane())
	nodeRunner.AppendPhase(phases.NewKubeletConfigPhase())

	// sets the data builder function, that will be used by the runner
	// both when running the entire workflow or single phases
	nodeRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) {
		return newNodeData(cmd, args, nodeOptions, out)
	})

	// binds the Runner to kubeadm upgrade node command by altering
	// command help, adding --skip-phases flag and by adding phases subcommands
	nodeRunner.BindToCommand(cmd)

	return cmd
}


func newNodeOptions() *nodeOptions

// newNodeOptions returns a struct ready for being used for creating cmd kubeadm upgrade node flags.
func newNodeOptions() *nodeOptions {
	return &nodeOptions{
		kubeConfigPath: constants.GetKubeletKubeConfigPath(),
		dryRun:         false,
		renewCerts:     true,
		etcdUpgrade:    true,
	}
}


func addUpgradeNodeFlags(flagSet *flag.FlagSet, nodeOptions *nodeOptions) {
	options.AddKubeConfigFlag(flagSet, &nodeOptions.kubeConfigPath)
	flagSet.BoolVar(&nodeOptions.dryRun, options.DryRun, nodeOptions.dryRun, "Do not change any state, just output the actions that would be performed.")
	flagSet.BoolVar(&nodeOptions.renewCerts, options.CertificateRenewal, nodeOptions.renewCerts, "Perform the renewal of certificates used by component changed during upgrades.")
	flagSet.BoolVar(&nodeOptions.etcdUpgrade, options.EtcdUpgrade, nodeOptions.etcdUpgrade, "Perform the upgrade of etcd.")
	flagSet.StringSliceVar(&nodeOptions.ignorePreflightErrors, options.IgnorePreflightErrors, nodeOptions.ignorePreflightErrors, "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.")
}


// newNodeData returns a new nodeData struct to be used for the execution of the kubeadm upgrade node workflow.
// This func takes care of validating nodeOptions passed to the command, and then it converts
// options into the internal InitConfiguration type that is used as input all the phases in the kubeadm upgrade node workflow
func newNodeData(cmd *cobra.Command, args []string, options *nodeOptions, out io.Writer) (*nodeData, error) {
	client, err := getClient(options.kubeConfigPath, options.dryRun)
	if err != nil {
		return nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", options.kubeConfigPath)
	}

	// isControlPlane checks if a node is a control-plane node by looking up
	// the kube-apiserver manifest file
	isControlPlaneNode := true
	filepath := constants.GetStaticPodFilepath(constants.KubeAPIServer, constants.GetStaticPodDirectory())
	if _, err := os.Stat(filepath); os.IsNotExist(err) {
		isControlPlaneNode = false
	}

	// Fetches the cluster configuration
	// NB in case of control-plane node, we are reading all the info for the node; in case of NOT control-plane node
	//    (worker node), we are not reading local API address and the CRI socket from the node object
	cfg, err := configutil.FetchInitConfigurationFromCluster(client, nil, "upgrade", !isControlPlaneNode, false)
	if err != nil {
		return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap")
	}

	ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors, cfg.NodeRegistration.IgnorePreflightErrors)
	if err != nil {
		return nil, err
	}
	// Also set the union of pre-flight errors to JoinConfiguration, to provide a consistent view of the runtime configuration:
	cfg.NodeRegistration.IgnorePreflightErrors = sets.List(ignorePreflightErrorsSet)
	return &nodeData{
		etcdUpgrade:           options.etcdUpgrade,
		renewCerts:            options.renewCerts,
		dryRun:                options.dryRun,
		cfg:                   cfg,
		client:                client,
		isControlPlaneNode:    isControlPlaneNode,
		patchesDir:            options.patchesDir,
		ignorePreflightErrors: ignorePreflightErrorsSet,
		kubeConfigPath:        options.kubeConfigPath,
		outputWriter:          out,
	}, nil
}


func (d *nodeData) DryRun() bool

// DryRun returns the dryRun flag.
func (d *nodeData) DryRun() bool {
	return d.dryRun
}


func (d *nodeData) EtcdUpgrade() bool

// EtcdUpgrade returns the etcdUpgrade flag.
func (d *nodeData) EtcdUpgrade() bool {
	return d.etcdUpgrade
}


func (d *nodeData) RenewCerts() bool

// RenewCerts returns the renewCerts flag.
func (d *nodeData) RenewCerts() bool {
	return d.renewCerts
}


// Cfg returns initConfiguration.
func (d *nodeData) Cfg() *kubeadmapi.InitConfiguration {
	return d.cfg
}


// IsControlPlaneNode returns the isControlPlaneNode flag.
func (d *nodeData) IsControlPlaneNode() bool {
	return d.isControlPlaneNode
}


// Client returns a Kubernetes client to be used by kubeadm.
func (d *nodeData) Client() clientset.Interface {
	return d.client
}


func (d *nodeData) PatchesDir() string

// PatchesDir returns the folder where patches for components are stored
func (d *nodeData) PatchesDir() string {
	return d.patchesDir
}


// IgnorePreflightErrors returns the list of preflight errors to ignore.
func (d *nodeData) IgnorePreflightErrors() sets.Set[string] {
	return d.ignorePreflightErrors
}


// KubeconfigPath returns the path to the user kubeconfig file.
func (d *nodeData) KubeConfigPath() string {
	return d.kubeConfigPath
}


func (d *nodeData) OutputWriter() io.Writer {
	return d.outputWriter
}


// newCmdPlan returns the cobra command for `kubeadm upgrade plan`
func newCmdPlan(apf *applyPlanFlags) *cobra.Command {
	flags := &planFlags{
		applyPlanFlags: apf,
	}

	outputFlags := newUpgradePlanPrintFlags(output.TextOutput)

	cmd := &cobra.Command{
		Use:   "plan [version] [flags]",
		Short: "Check which versions are available to upgrade to and validate whether your current cluster is upgradeable. To skip the internet check, pass in the optional [version] parameter",
		RunE: func(_ *cobra.Command, args []string) error {
			printer, err := outputFlags.ToPrinter()
			if err != nil {
				return errors.Wrap(err, "could not construct output printer")
			}

			return runPlan(flags, args, printer)
		},
	}

	outputFlags.AddFlags(cmd)

	// Register the common flags for apply and plan
	addApplyPlanFlags(cmd.Flags(), flags.applyPlanFlags)
	return cmd
}


// newComponentUpgradePlan helper creates outputapiv1alpha2.ComponentUpgradePlan object
func newComponentUpgradePlan(name, currentVersion, newVersion string) outputapiv1alpha2.ComponentUpgradePlan {
	return outputapiv1alpha2.ComponentUpgradePlan{
		Name:           name,
		CurrentVersion: currentVersion,
		NewVersion:     newVersion,
	}
}


func newUpgradePlanPrintFlags(outputFormat string) *upgradePlanPrintFlags {
	return &upgradePlanPrintFlags{
		JSONYamlPrintFlags: &upgradePlanJSONYamlPrintFlags{},
		TextPrintFlags:     &upgradePlanTextPrintFlags{},
		TypeSetterPrinter:  printers.NewTypeSetter(outputapischeme.Scheme),
		OutputFormat:       strings.ToLower(outputFormat),
	}
}


// AllowedFormats returns a list of allowed output formats
func (pf *upgradePlanPrintFlags) AllowedFormats() []string {
	ret := pf.TextPrintFlags.AllowedFormats()
	return append(ret, pf.JSONYamlPrintFlags.AllowedFormats()...)
}


// AddFlags receives a *cobra.Command reference and binds
// flags related to Kubeadm printing to it
func (pf *upgradePlanPrintFlags) AddFlags(cmd *cobra.Command) {
	pf.TextPrintFlags.AddFlags(cmd)
	pf.JSONYamlPrintFlags.AddFlags(cmd)
	// TODO: once we are confident the feature is graduated we should remove the EXPERIMENTAL text below:
	// https://github.com/kubernetes/kubeadm/issues/494
	cmd.Flags().StringVarP(&pf.OutputFormat, "output", "o", pf.OutputFormat, fmt.Sprintf("EXPERIMENTAL: Output format. One of: %s.", strings.Join(pf.AllowedFormats(), "|")))
}


// ToPrinter receives an outputFormat and returns a printer capable of
// handling format printing.
// Returns error if the specified outputFormat does not match supported formats.
func (pf *upgradePlanPrintFlags) ToPrinter() (output.Printer, error) {
	switch pf.OutputFormat {
	case output.TextOutput:
		return pf.TextPrintFlags.ToPrinter(pf.OutputFormat)
	case output.JSONOutput:
		return newUpgradePlanJSONYAMLPrinter(pf.TypeSetterPrinter.WrapToPrinter(pf.JSONYamlPrintFlags.ToPrinter(output.JSONOutput)))
	case output.YAMLOutput:
		return newUpgradePlanJSONYAMLPrinter(pf.TypeSetterPrinter.WrapToPrinter(pf.JSONYamlPrintFlags.ToPrinter(output.YAMLOutput)))
	default:
		return nil, genericclioptions.NoCompatiblePrinterError{OutputFormat: &pf.OutputFormat, AllowedFormats: pf.AllowedFormats()}
	}
}


// AllowedFormats returns a list of allowed output formats
func (pf *upgradePlanJSONYamlPrintFlags) AllowedFormats() []string {
	return []string{output.JSONOutput, output.YAMLOutput}
}


// newUpgradePlanJSONYAMLPrinter creates a new upgradePlanJSONYAMLPrinter object
func newUpgradePlanJSONYAMLPrinter(resourcePrinter printers.ResourcePrinter, err error) (output.Printer, error) {
	if err != nil {
		return nil, err
	}
	return &upgradePlanJSONYAMLPrinter{ResourcePrinterWrapper: output.ResourcePrinterWrapper{Printer: resourcePrinter}}, nil
}


// PrintObj is an implementation of ResourcePrinter.PrintObj that adds object to the buffer
func (p *upgradePlanJSONYAMLPrinter) PrintObj(obj runtime.Object, writer io.Writer) error {
	item, ok := obj.(*outputapiv1alpha2.ComponentUpgradePlan)
	if !ok {
		return errors.Errorf("expected ComponentUpgradePlan, but got %+v", obj)
	}
	p.Buffer = append(p.Buffer, *item)
	return nil
}


// Flush writes any buffered data once last object is added
func (p *upgradePlanJSONYAMLPrinter) Flush(writer io.Writer, last bool) {
	if !last {
		return
	}
	if len(p.Buffer) == 0 {
		return
	}
	plan := &outputapiv1alpha2.UpgradePlan{Components: p.Buffer}
	if err := p.Printer.PrintObj(plan, writer); err != nil {
		fmt.Fprintf(os.Stderr, "could not flush output buffer: %v\n", err)
	}
}


// Close empties the list of buffered components
func (p *upgradePlanJSONYAMLPrinter) Close(writer io.Writer) {
	p.Buffer = p.Buffer[:0]
}


// Flush writes any buffered data
func (p *upgradePlanTextPrinter) Flush(writer io.Writer, last bool) {
	if p.tabwriter != nil {
		p.tabwriter.Flush()
		p.tabwriter = nil
		p.Fprintln(writer, "")
	}
}


// PrintObj is an implementation of ResourcePrinter.PrintObj for upgrade plan plain text output
func (p *upgradePlanTextPrinter) PrintObj(obj runtime.Object, writer io.Writer) error {
	if p.tabwriter == nil {
		p.tabwriter = tabwriter.NewWriter(writer, 10, 4, 3, ' ', 0)
		// Print header
		fmt.Fprintln(p.tabwriter, strings.Join(p.columns, "\t"))
	}

	item, ok := obj.(*outputapiv1alpha2.ComponentUpgradePlan)
	if !ok {
		return errors.Errorf("expected ComponentUpgradePlan, but got %+v", obj)
	}

	// Print item
	fmt.Fprintf(p.tabwriter, "%s\t%s\t%s\n", item.Name, item.CurrentVersion, item.NewVersion)
	return nil
}


func (pf *upgradePlanTextPrintFlags) AddFlags(cmd *cobra.Command) {}


// AllowedFormats returns a list of allowed output formats
func (pf *upgradePlanTextPrintFlags) AllowedFormats() []string {
	return []string{output.TextOutput}
}


// ToPrinter returns a kubeadm printer for the text output format
func (pf *upgradePlanTextPrintFlags) ToPrinter(outputFormat string) (output.Printer, error) {
	if outputFormat == output.TextOutput {
		return &upgradePlanTextPrinter{columns: []string{"COMPONENT", "CURRENT", "TARGET"}}, nil
	}
	return nil, genericclioptions.NoCompatiblePrinterError{OutputFormat: &outputFormat, AllowedFormats: []string{output.JSONOutput, output.YAMLOutput, output.TextOutput}}
}


// runPlan takes care of outputting available versions to upgrade to for the user
func runPlan(flags *planFlags, args []string, printer output.Printer) error {
	// Start with the basics, verify that the cluster is healthy, build a client and a versionGetter. Never dry-run when planning.
	klog.V(1).Infoln("[upgrade/plan] verifying health of cluster")
	klog.V(1).Infoln("[upgrade/plan] retrieving configuration from cluster")
	client, versionGetter, cfg, err := enforceRequirements(flags.applyPlanFlags, args, false, false, printer, loadConfig)
	if err != nil {
		return err
	}

	// Currently this is the only method we have for distinguishing
	// external etcd vs static pod etcd
	isExternalEtcd := cfg.Etcd.External != nil

	// Compute which upgrade possibilities there are
	klog.V(1).Infoln("[upgrade/plan] computing upgrade possibilities")
	availUpgrades, err := upgrade.GetAvailableUpgrades(versionGetter, flags.allowExperimentalUpgrades, flags.allowRCUpgrades, isExternalEtcd, client, constants.GetStaticPodDirectory(), printer)
	if err != nil {
		return errors.Wrap(err, "[upgrade/versions] FATAL")
	}

	// Fetch the current state of the component configs
	klog.V(1).Infoln("[upgrade/plan] analysing component config version states")
	configVersionStates, err := getComponentConfigVersionStates(&cfg.ClusterConfiguration, client, flags.cfgPath)
	if err != nil {
		return errors.WithMessage(err, "[upgrade/versions] FATAL")
	}

	// No upgrades available
	if len(availUpgrades) == 0 {
		klog.V(1).Infoln("[upgrade/plan] Awesome, you're up-to-date! Enjoy!")
		return nil
	}

	// Generate and print upgrade plans
	for _, up := range availUpgrades {
		plan, unstableVersionFlag, err := genUpgradePlan(&up, isExternalEtcd)
		if err != nil {
			return err
		}

		// Actually, this is needed for machine-readable output only.
		// printUpgradePlan won't output the configVersionStates as it will simply print the same table several times
		// in the human-readable output if it did so
		plan.ConfigVersions = configVersionStates

		printUpgradePlan(&up, plan, unstableVersionFlag, isExternalEtcd, os.Stdout, printer)
	}

	// Finally, print the component config state table
	printComponentConfigVersionStates(configVersionStates, os.Stdout, printer)
	return nil
}


// TODO There is currently no way to cleanly output upgrades that involve adding, removing, or changing components
// https://github.com/kubernetes/kubeadm/issues/810 was created to track addressing this.
func appendDNSComponent(components []outputapiv1alpha2.ComponentUpgradePlan, up *upgrade.Upgrade, name string) []outputapiv1alpha2.ComponentUpgradePlan {
	beforeVersion := up.Before.DNSVersion
	afterVersion := up.After.DNSVersion

	if beforeVersion != "" || afterVersion != "" {
		components = append(components, newComponentUpgradePlan(name, beforeVersion, afterVersion))
	}
	return components
}


// genUpgradePlan generates output-friendly upgrade plan out of upgrade.Upgrade structure
func genUpgradePlan(up *upgrade.Upgrade, isExternalEtcd bool) (*outputapiv1alpha2.UpgradePlan, string, error) {
	newK8sVersion, err := version.ParseSemantic(up.After.KubeVersion)
	if err != nil {
		return nil, "", errors.Wrapf(err, "Unable to parse normalized version %q as a semantic version", up.After.KubeVersion)
	}

	unstableVersionFlag := ""
	if len(newK8sVersion.PreRelease()) != 0 {
		if strings.HasPrefix(newK8sVersion.PreRelease(), "rc") {
			unstableVersionFlag = " --allow-release-candidate-upgrades"
		} else {
			unstableVersionFlag = " --allow-experimental-upgrades"
		}
	}

	components := []outputapiv1alpha2.ComponentUpgradePlan{}

	if up.CanUpgradeKubelets() {
		// The map is of the form <old-version>:<node-count>. Here all the keys are put into a slice and sorted
		// in order to always get the right order. Then the map value is extracted separately
		for _, oldVersion := range sortedSliceFromStringIntMap(up.Before.KubeletVersions) {
			nodeCount := up.Before.KubeletVersions[oldVersion]
			components = append(components, newComponentUpgradePlan(constants.Kubelet, fmt.Sprintf("%d x %s", nodeCount, oldVersion), up.After.KubeVersion))
		}
	}

	components = append(components, newComponentUpgradePlan(constants.KubeAPIServer, up.Before.KubeVersion, up.After.KubeVersion))
	components = append(components, newComponentUpgradePlan(constants.KubeControllerManager, up.Before.KubeVersion, up.After.KubeVersion))
	components = append(components, newComponentUpgradePlan(constants.KubeScheduler, up.Before.KubeVersion, up.After.KubeVersion))
	components = append(components, newComponentUpgradePlan(constants.KubeProxy, up.Before.KubeVersion, up.After.KubeVersion))

	components = appendDNSComponent(components, up, constants.CoreDNS)

	if !isExternalEtcd {
		components = append(components, newComponentUpgradePlan(constants.Etcd, up.Before.EtcdVersion, up.After.EtcdVersion))
	}

	return &outputapiv1alpha2.UpgradePlan{Components: components}, unstableVersionFlag, nil
}


func getComponentConfigVersionStates(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, cfgPath string) ([]outputapiv1alpha2.ComponentConfigVersionState, error) {
	docmap := kubeadmapi.DocumentMap{}

	if cfgPath != "" {
		bytes, err := os.ReadFile(cfgPath)
		if err != nil {
			return nil, errors.Wrapf(err, "unable to read config file %q", cfgPath)
		}

		docmap, err = kubeadmutil.SplitYAMLDocuments(bytes)
		if err != nil {
			return nil, err
		}
	}

	return componentconfigs.GetVersionStates(cfg, client, docmap)
}


// printUpgradePlan prints a UX-friendly overview of what versions are available to upgrade to
func printUpgradePlan(up *upgrade.Upgrade, plan *outputapiv1alpha2.UpgradePlan, unstableVersionFlag string, isExternalEtcd bool, writer io.Writer, printer output.Printer) {
	printHeader := true
	printManualUpgradeHeader := true
	for _, component := range plan.Components {
		if isExternalEtcd && component.Name == constants.Etcd {
			// Don't print etcd if it's external
			continue
		} else if component.Name == constants.Kubelet {
			if printManualUpgradeHeader {
				printer.Fprintln(writer, "Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':")
				plan := newComponentUpgradePlan(component.Name, component.CurrentVersion, component.NewVersion)
				printer.PrintObj(&plan, writer)
				printManualUpgradeHeader = false
			} else {
				plan := newComponentUpgradePlan("", component.CurrentVersion, component.NewVersion)
				printer.PrintObj(&plan, writer)
			}
		} else {
			if printHeader {
				// End of manual upgrades table
				printer.Flush(writer, false)
				printer.Fprintf(writer, "Upgrade to the latest %s:\n", up.Description)
				printer.Fprintln(writer, "")
				printHeader = false
			}
			plan := newComponentUpgradePlan(component.Name, component.CurrentVersion, component.NewVersion)
			printer.PrintObj(&plan, writer)
		}
	}
	printer.Flush(writer, true)

	printer.Fprintln(writer, "You can now apply the upgrade by executing the following command:")
	printer.Fprintln(writer, "")
	printer.Fprintf(writer, "\tkubeadm upgrade apply %s%s\n", up.After.KubeVersion, unstableVersionFlag)
	printer.Fprintln(writer, "")

	if up.Before.KubeadmVersion != up.After.KubeadmVersion {
		printer.Fprintf(writer, "Note: Before you can perform this upgrade, you have to update kubeadm to %s.\n", up.After.KubeadmVersion)
		printer.Fprintln(writer, "")
	}

	printLineSeparator(writer, printer)
	printer.Close(writer)
}


// sortedSliceFromStringIntMap returns a slice of the keys in the map sorted alphabetically
func sortedSliceFromStringIntMap(strMap map[string]uint16) []string {
	strSlice := []string{}
	for k := range strMap {
		strSlice = append(strSlice, k)
	}
	sort.Strings(strSlice)
	return strSlice
}


func strOrDash(s string) string

func strOrDash(s string) string {
	if s != "" {
		return s
	}
	return "-"
}


func yesOrNo(b bool) string

func yesOrNo(b bool) string {
	if b {
		return "yes"
	}
	return "no"
}


func printLineSeparator(w io.Writer, printer output.Printer)


func printComponentConfigVersionStates(versionStates []outputapiv1alpha2.ComponentConfigVersionState, w io.Writer, printer output.Printer) {
	if len(versionStates) == 0 {
		printer.Fprintln(w, "No information available on component configs.")
		return
	}

	printer.Fprintln(w, dedent.Dedent(`
		The table below shows the current state of component configs as understood by this version of kubeadm.
		Configs that have a "yes" mark in the "MANUAL UPGRADE REQUIRED" column require manual config upgrade or
		resetting to kubeadm defaults before a successful upgrade can be performed. The version to manually
		upgrade to is denoted in the "PREFERRED VERSION" column.
	`))

	tabw := tabwriter.NewWriter(w, 10, 4, 3, ' ', 0)
	printer.Fprintln(tabw, "API GROUP\tCURRENT VERSION\tPREFERRED VERSION\tMANUAL UPGRADE REQUIRED")

	for _, state := range versionStates {
		printer.Fprintf(tabw,
			"%s\t%s\t%s\t%s\n",
			state.Group,
			strOrDash(state.CurrentVersion),
			strOrDash(state.PreferredVersion),
			yesOrNo(state.ManualUpgradeRequired),
		)
	}

	tabw.Flush()
	printLineSeparator(w, printer)
}


// NewCmdUpgrade returns the cobra command for `kubeadm upgrade`
func NewCmdUpgrade(out io.Writer) *cobra.Command {
	flags := &applyPlanFlags{
		kubeConfigPath:            kubeadmconstants.GetAdminKubeConfigPath(),
		cfgPath:                   "",
		featureGatesString:        "",
		allowExperimentalUpgrades: false,
		allowRCUpgrades:           false,
		printConfig:               false,
		out:                       out,
	}

	cmd := &cobra.Command{
		Use:   "upgrade",
		Short: "Upgrade your cluster smoothly to a newer version with this command",
		Run:   cmdutil.SubCmdRun(),
	}

	cmd.AddCommand(newCmdApply(flags))
	cmd.AddCommand(newCmdPlan(flags))
	cmd.AddCommand(newCmdDiff(out))
	cmd.AddCommand(newCmdNode(out))
	return cmd
}


func addApplyPlanFlags(fs *pflag.FlagSet, flags *applyPlanFlags) {
	options.AddKubeConfigFlag(fs, &flags.kubeConfigPath)
	options.AddConfigFlag(fs, &flags.cfgPath)

	fs.BoolVar(&flags.allowExperimentalUpgrades, "allow-experimental-upgrades", flags.allowExperimentalUpgrades, "Show unstable versions of Kubernetes as an upgrade alternative and allow upgrading to an alpha/beta/release candidate versions of Kubernetes.")
	fs.BoolVar(&flags.allowRCUpgrades, "allow-release-candidate-upgrades", flags.allowRCUpgrades, "Show release candidate versions of Kubernetes as an upgrade alternative and allow upgrading to a release candidate versions of Kubernetes.")
	fs.BoolVar(&flags.printConfig, "print-config", flags.printConfig, "Specifies whether the configuration file that will be used in the upgrade should be printed or not.")
	options.AddFeatureGatesStringFlag(fs, &flags.featureGatesString)
	options.AddIgnorePreflightErrorsFlag(fs, &flags.ignorePreflightErrors)
}


// SubCmdRun returns a function that handles a case where a subcommand must be specified
// Without this callback, if a user runs just the command without a subcommand,
// or with an invalid subcommand, cobra will print usage information, but still exit cleanly.
func SubCmdRun() func(c *cobra.Command, args []string) {
	return func(c *cobra.Command, args []string) {
		if len(args) > 0 {
			kubeadmutil.CheckErr(usageErrorf(c, "invalid subcommand %q", strings.Join(args, " ")))
		}
		c.Help()
		kubeadmutil.CheckErr(kubeadmutil.ErrExit)
	}
}


func usageErrorf(c *cobra.Command, format string, args ...interface{}) error {
	msg := fmt.Sprintf(format, args...)
	return errors.Errorf("%s\nSee '%s -h' for help and examples", msg, c.CommandPath())
}


// ValidateExactArgNumber validates that the required top-level arguments are specified
func ValidateExactArgNumber(args []string, supportedArgs []string) error {
	lenSupported := len(supportedArgs)
	validArgs := 0
	// Disregard possible "" arguments; they are invalid
	for _, arg := range args {
		if len(arg) > 0 {
			validArgs++
		}
		// break early for too many arguments
		if validArgs > lenSupported {
			return errors.Errorf("too many arguments. Required arguments: %v", supportedArgs)
		}
	}

	if validArgs < lenSupported {
		return errors.Errorf("missing one or more required arguments. Required arguments: %v", supportedArgs)
	}
	return nil
}


// GetKubeConfigPath can be used to search for a kubeconfig in standard locations
// if and empty string is passed to the function. If a non-empty string is passed
// the function returns the same string.
func GetKubeConfigPath(file string) string {
	// If a value is provided respect that.
	if file != "" {
		return file
	}
	// Find a config in the standard locations using DefaultClientConfigLoadingRules,
	// but also consider the default config path.
	rules := clientcmd.NewDefaultClientConfigLoadingRules()
	rules.Precedence = append(rules.Precedence, kubeadmconstants.GetAdminKubeConfigPath())
	file = rules.GetDefaultFilename()
	klog.V(1).Infof("Using kubeconfig file: %s", file)
	return file
}


// AddCRISocketFlag adds the cri-socket flag to the supplied flagSet
func AddCRISocketFlag(flagSet *pflag.FlagSet, criSocket *string) {
	flagSet.StringVar(
		criSocket, options.NodeCRISocket, *criSocket,
		"Path to the CRI socket to connect. If empty kubeadm will try to auto-detect this value; use this option only if you have more than one CRI installed or if you have non-standard CRI socket.",
	)
}


// DefaultInitConfiguration return default InitConfiguration. Avoid running the CRI auto-detection
// code as we don't need it.
func DefaultInitConfiguration() *kubeadmapiv1.InitConfiguration {
	initCfg := &kubeadmapiv1.InitConfiguration{
		NodeRegistration: kubeadmapiv1.NodeRegistrationOptions{
			CRISocket: kubeadmconstants.UnknownCRISocket, // avoid CRI detection
		},
	}
	return initCfg
}


// InteractivelyConfirmAction asks the user whether they _really_ want to take the action.
func InteractivelyConfirmAction(action, question string, r io.Reader) error {
	fmt.Printf("[%s] %s [y/N]: ", action, question)
	scanner := bufio.NewScanner(r)
	scanner.Scan()
	if err := scanner.Err(); err != nil {
		return errors.Wrap(err, "couldn't read from standard input")
	}
	answer := scanner.Text()
	if strings.EqualFold(answer, "y") || strings.EqualFold(answer, "yes") {
		return nil
	}

	return errors.New("won't proceed; the user didn't answer (Y|y) in order to continue")
}


// GetClientSet gets a real or fake client depending on whether the user is dry-running or not
func GetClientSet(file string, dryRun bool) (clientset.Interface, error) {
	if dryRun {
		dryRunGetter, err := apiclient.NewClientBackedDryRunGetterFromKubeconfig(file)
		if err != nil {
			return nil, err
		}
		return apiclient.NewDryRunClient(dryRunGetter, os.Stdout), nil
	}
	return kubeconfigutil.ClientSetFromFile(file)
}


func LongDesc(s string) string

// LongDesc is designed to help with producing better long command line descriptions in code.
// Its behavior is somewhat inspired by the same function of kubectl, which uses Markdown for the input message.
// This one is not Markdown compliant, but it covers the needs of kubeadm. In particular:
// - Beginning and trailing space characters (including empty lines), are stripped from the output.
// - Consecutive non-empty lines of text are joined with spaces to form paragraphs.
// - Paragraphs are blocks of text divided by one or more empty lines or lines consisting only of "space" characters.
// - Paragraphs are spaced by precisely one empty line in the output.
// - A line break can be forced by adding a couple of empty spaces at the end of a text line.
// - All indentation is removed. The resulting output is not indented.
func LongDesc(s string) string {
	// Strip beginning and trailing space characters (including empty lines) and split the lines into a slice
	lines := strings.Split(strings.TrimSpace(s), "\n")

	output := []string{}
	paragraph := []string{}

	for _, line := range lines {
		// Remove indentation and trailing spaces from the current line
		trimmedLine := strings.TrimSpace(line)
		if trimmedLine == "" {
			if len(paragraph) > 0 {
				// If the line is empty and the current paragraph is not, we have reached a paragraph end.
				// (if the paragraph and the line are empty, then this is non-first empty line in between paragraphs and needs to be ignored)
				// In that case we join all of the paragraph lines with a single space,
				// add a trailing newline character (to ensure an empty line after the paragraph),
				// append the paragraph text to the output and clear everything in the current paragraph slice.
				output = append(output, strings.Join(paragraph, " ")+"\n")
				paragraph = []string{}
			}
		} else {
			// Non-empty text line, append it to the current paragraph
			paragraph = append(paragraph, trimmedLine)
			if strings.HasSuffix(line, "  ") {
				// If the original line has a suffix of couple of spaces, then we add a line break.
				// This is achieved by flushing out the current paragraph and starting a new one.
				// No trailing space is added to the flushed paragraph,
				// so that no empty line is placed between the old and the new paragraphs (a simple line break)
				output = append(output, strings.Join(paragraph, " "))
				paragraph = []string{}
			}
		}
	}

	// The last paragraph is always unflushed, so flush it.
	// We don't add a trailing newline character, so that we won't have to strip the output.
	output = append(output, strings.Join(paragraph, " "))

	// Join all the paragraphs together with new lines in between them.
	return strings.Join(output, "\n")
}


func Examples(s string) string

// Examples is designed to help with producing examples for command line usage.
// Its behavior is mimicking a similar kubectl function in the following ways:
// - Beginning and trailing space characters (including empty lines), are stripped from the output.
// - All lines of text are stripped of beginning and trailing spaces (thus loosing indentation) and are then double-space indented.
func Examples(s string) string {
	trimmedText := strings.TrimSpace(s)
	if trimmedText == "" {
		return ""
	}

	const indent = `  `
	inLines := strings.Split(trimmedText, "\n")
	outLines := make([]string, 0, len(inLines))

	for _, line := range inLines {
		outLines = append(outLines, indent+strings.TrimSpace(line))
	}

	return strings.Join(outLines, "\n")
}


// GetJoinWorkerCommand returns the kubeadm join command for a given token and
// Kubernetes cluster (the current cluster in the kubeconfig file)
func GetJoinWorkerCommand(kubeConfigFile, token string, skipTokenPrint bool) (string, error) {
	return getJoinCommand(kubeConfigFile, token, "", false, skipTokenPrint, false)
}


// GetJoinControlPlaneCommand returns the kubeadm join command for a given token and
// Kubernetes cluster (the current cluster in the kubeconfig file)
func GetJoinControlPlaneCommand(kubeConfigFile, token, key string, skipTokenPrint, skipCertificateKeyPrint bool) (string, error) {
	return getJoinCommand(kubeConfigFile, token, key, true, skipTokenPrint, skipCertificateKeyPrint)
}


func getJoinCommand(kubeConfigFile, token, key string, controlPlane, skipTokenPrint, skipCertificateKeyPrint bool) (string, error) {
	// load the kubeconfig file to get the CA certificate and endpoint
	config, err := clientcmd.LoadFromFile(kubeConfigFile)
	if err != nil {
		return "", errors.Wrap(err, "failed to load kubeconfig")
	}

	// load the default cluster config
	_, clusterConfig := kubeconfigutil.GetClusterFromKubeConfig(config)
	if clusterConfig == nil {
		return "", errors.New("failed to get default cluster config")
	}

	// load CA certificates from the kubeconfig (either from PEM data or by file path)
	var caCerts []*x509.Certificate
	if clusterConfig.CertificateAuthorityData != nil {
		caCerts, err = clientcertutil.ParseCertsPEM(clusterConfig.CertificateAuthorityData)
		if err != nil {
			return "", errors.Wrap(err, "failed to parse CA certificate from kubeconfig")
		}
	} else if clusterConfig.CertificateAuthority != "" {
		caCerts, err = clientcertutil.CertsFromFile(clusterConfig.CertificateAuthority)
		if err != nil {
			return "", errors.Wrap(err, "failed to load CA certificate referenced by kubeconfig")
		}
	} else {
		return "", errors.New("no CA certificates found in kubeconfig")
	}

	// hash all the CA certs and include their public key pins as trusted values
	publicKeyPins := make([]string, 0, len(caCerts))
	for _, caCert := range caCerts {
		publicKeyPins = append(publicKeyPins, pubkeypin.Hash(caCert))
	}

	ctx := map[string]interface{}{
		"Token":                token,
		"CAPubKeyPins":         publicKeyPins,
		"ControlPlaneHostPort": strings.Replace(clusterConfig.Server, "https://", "", -1),
		"CertificateKey":       key,
		"ControlPlane":         controlPlane,
	}

	if skipTokenPrint {
		ctx["Token"] = template.HTML("<value withheld>")
	}
	if skipCertificateKeyPrint {
		ctx["CertificateKey"] = template.HTML("<value withheld>")
	}

	var out bytes.Buffer
	err = joinCommandTemplate.Execute(&out, ctx)
	if err != nil {
		return "", errors.Wrap(err, "failed to render join command template")
	}
	return out.String(), nil
}


// newCmdVersion provides the version information of kubeadm.
func newCmdVersion(out io.Writer) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "version",
		Short: "Print the version of kubeadm",
		RunE: func(cmd *cobra.Command, args []string) error {
			return RunVersion(out, cmd)
		},
		Args: cobra.NoArgs,
	}
	cmd.Flags().StringP("output", "o", "", "Output format; available options are 'yaml', 'json' and 'short'")
	return cmd
}


// RunVersion provides the version information of kubeadm in format depending on arguments
// specified in cobra.Command.
func RunVersion(out io.Writer, cmd *cobra.Command) error {
	klog.V(1).Infoln("[version] retrieving version info")
	clientVersion := version.Get()
	v := Version{
		ClientVersion: &clientVersion,
	}

	const flag = "output"
	of, err := cmd.Flags().GetString(flag)
	if err != nil {
		return errors.Wrapf(err, "error accessing flag %s for command %s", flag, cmd.Name())
	}

	switch of {
	case "":
		fmt.Fprintf(out, "kubeadm version: %#v\n", v.ClientVersion)
	case "short":
		fmt.Fprintf(out, "%s\n", v.ClientVersion.GitVersion)
	case "yaml":
		y, err := yaml.Marshal(&v)
		if err != nil {
			return err
		}
		fmt.Fprintln(out, string(y))
	case "json":
		y, err := json.MarshalIndent(&v, "", "  ")
		if err != nil {
			return err
		}
		fmt.Fprintln(out, string(y))
	default:
		return errors.Errorf("invalid output format: %s", of)
	}

	return nil
}


// ChecksumForConfigMap calculates a checksum for the supplied config map. The exact algorithm depends on hash and prefix parameters
func ChecksumForConfigMap(cm *v1.ConfigMap) string {
	hash := sha256.New()

	// Since maps are not ordered we need to make sure we order them somehow so we'll always get the same checksums
	// for the same config maps. The solution here is to extract the keys into a slice and sort them.
	// Then iterate over that slice to fetch the values to be hashed.
	keys := []string{}
	for key := range cm.Data {
		keys = append(keys, key)
	}
	sort.Strings(keys)

	for _, key := range keys {
		hash.Write([]byte(cm.Data[key]))
	}

	// Do the same as above, but for binaryData this time.
	keys = []string{}
	for key := range cm.BinaryData {
		keys = append(keys, key)
	}
	sort.Strings(keys)

	for _, key := range keys {
		hash.Write(cm.BinaryData[key])
	}

	return fmt.Sprintf("sha256:%x", hash.Sum(nil))
}


func SignConfigMap(cm *v1.ConfigMap)

// SignConfigMap calculates the supplied config map checksum and annotates it with it
func SignConfigMap(cm *v1.ConfigMap) {
	if cm.Annotations == nil {
		cm.Annotations = map[string]string{}
	}
	cm.Annotations[constants.ComponentConfigHashAnnotationKey] = ChecksumForConfigMap(cm)
}


// VerifyConfigMapSignature returns true if the config map has checksum annotation and it matches; false otherwise
func VerifyConfigMapSignature(cm *v1.ConfigMap) bool {
	signature, ok := cm.Annotations[constants.ComponentConfigHashAnnotationKey]
	if !ok {
		return false
	}
	return signature == ChecksumForConfigMap(cm)
}


// FromDocumentMap looks in the document map for documents with this handler's group.
// If such are found a new component config is instantiated and the documents are loaded into it.
// No error is returned if no documents are found.
func (h *handler) FromDocumentMap(docmap kubeadmapi.DocumentMap) (kubeadmapi.ComponentConfig, error) {
	for gvk := range docmap {
		if gvk.Group == h.GroupVersion.Group {
			cfg := h.CreateEmpty()
			if err := cfg.Unmarshal(docmap); err != nil {
				return nil, err
			}
			// consider all successfully loaded configs from a document map as user supplied
			cfg.SetUserSupplied(true)
			return cfg, nil
		}
	}
	return nil, nil
}


// fromConfigMap is an utility function, which will load the value of a key of a config map and use h.FromDocumentMap() to perform the parsing
// This is an utility func. Used by the component config support implementations. Don't use it outside of that context.
func (h *handler) fromConfigMap(client clientset.Interface, cmName, cmKey string, mustExist bool) (kubeadmapi.ComponentConfig, error) {
	configMap, err := apiclient.GetConfigMapWithRetry(client, metav1.NamespaceSystem, cmName)
	if err != nil {
		if !mustExist && (apierrors.IsNotFound(err) || apierrors.IsForbidden(err)) {
			klog.Warningf("Warning: No %s config is loaded. Continuing without it: %v", h.GroupVersion, err)
			return nil, nil
		}
		return nil, err
	}

	configData, ok := configMap.Data[cmKey]
	if !ok {
		return nil, errors.Errorf("unexpected error when reading %s ConfigMap: %s key value pair missing", cmName, cmKey)
	}

	gvkmap, err := kubeadmutil.SplitYAMLDocuments([]byte(configData))
	if err != nil {
		return nil, err
	}

	// If the checksum comes up neatly we assume the config was generated
	generatedConfig := VerifyConfigMapSignature(configMap)

	componentCfg, err := h.FromDocumentMap(gvkmap)
	if err != nil {
		// If the config was generated and we get UnsupportedConfigVersionError, we skip loading it.
		// This will force us to use the generated default current version (effectively regenerating the config with the current version).
		if _, ok := err.(*UnsupportedConfigVersionError); ok && generatedConfig {
			return nil, nil
		}
		return nil, err
	}

	if componentCfg != nil {
		componentCfg.SetUserSupplied(!generatedConfig)
	}

	return componentCfg, nil
}


// FromCluster loads a component from a config map in the cluster
func (h *handler) FromCluster(clientset clientset.Interface, clusterCfg *kubeadmapi.ClusterConfiguration) (kubeadmapi.ComponentConfig, error) {
	return h.fromCluster(h, clientset, clusterCfg)
}


func (cb *configBase) IsUserSupplied() bool {
	return cb.userSupplied
}


func (cb *configBase) SetUserSupplied(userSupplied bool) {
	cb.userSupplied = userSupplied
}


func (cb *configBase) DeepCopyInto(other *configBase) {
	*other = *cb
}


func cloneBytes(in []byte) []byte

func cloneBytes(in []byte) []byte {
	out := make([]byte, len(in))
	copy(out, in)
	return out
}


// Marshal is an utility function, used by the component config support implementations to marshal a runtime.Object to YAML with the
// correct group and version
func (cb *configBase) Marshal(object runtime.Object) ([]byte, error) {
	return kubeadmutil.MarshalToYamlForCodecs(object, cb.GroupVersion, Codecs)
}


// Unmarshal attempts to unmarshal a runtime.Object from a document map. If no object is found, no error is returned.
// If a matching group is found, but no matching version an error is returned indicating that users should do manual conversion.
func (cb *configBase) Unmarshal(from kubeadmapi.DocumentMap, into runtime.Object) error {
	for gvk, yaml := range from {
		// If this is a different group, we ignore it
		if gvk.Group != cb.GroupVersion.Group {
			continue
		}

		if gvk.Version != cb.GroupVersion.Version {
			return &UnsupportedConfigVersionError{
				OldVersion:     gvk.GroupVersion(),
				CurrentVersion: cb.GroupVersion,
				Document:       cloneBytes(yaml),
			}
		}

		// Print warnings for strict errors
		if err := strict.VerifyUnmarshalStrict([]*runtime.Scheme{Scheme}, gvk, yaml); err != nil {
			klog.Warning(err.Error())
		}

		// As long as we support only component configs with a single kind, this is allowed
		return runtime.DecodeInto(Codecs.UniversalDecoder(), yaml, into)
	}

	return nil
}


// ensureInitializedComponentConfigs is an utility func to initialize the ComponentConfigMap in ClusterConfiguration prior to possible writes to it
func ensureInitializedComponentConfigs(clusterCfg *kubeadmapi.ClusterConfiguration) {
	if clusterCfg.ComponentConfigs == nil {
		clusterCfg.ComponentConfigs = kubeadmapi.ComponentConfigMap{}
	}
}


// Default sets up defaulted component configs in the supplied ClusterConfiguration
func Default(clusterCfg *kubeadmapi.ClusterConfiguration, localAPIEndpoint *kubeadmapi.APIEndpoint, nodeRegOpts *kubeadmapi.NodeRegistrationOptions) {
	ensureInitializedComponentConfigs(clusterCfg)

	for _, handler := range known {
		// If the component config exists, simply default it. Otherwise, create it before defaulting.
		group := handler.GroupVersion.Group
		if componentCfg, ok := clusterCfg.ComponentConfigs[group]; ok {
			componentCfg.Default(clusterCfg, localAPIEndpoint, nodeRegOpts)
		} else {
			componentCfg := handler.CreateEmpty()
			componentCfg.Default(clusterCfg, localAPIEndpoint, nodeRegOpts)
			clusterCfg.ComponentConfigs[group] = componentCfg
		}
	}
}


// FetchFromCluster attempts to fetch all known component configs from their config maps and store them in the supplied ClusterConfiguration
func FetchFromCluster(clusterCfg *kubeadmapi.ClusterConfiguration, client clientset.Interface) error {
	ensureInitializedComponentConfigs(clusterCfg)

	for _, handler := range known {
		componentCfg, err := handler.FromCluster(client, clusterCfg)
		if err != nil {
			return err
		}

		if componentCfg != nil {
			clusterCfg.ComponentConfigs[handler.GroupVersion.Group] = componentCfg
		}
	}

	return nil
}


// FetchFromDocumentMap attempts to load all known component configs from a document map into the supplied ClusterConfiguration
func FetchFromDocumentMap(clusterCfg *kubeadmapi.ClusterConfiguration, docmap kubeadmapi.DocumentMap) error {
	ensureInitializedComponentConfigs(clusterCfg)

	for _, handler := range known {
		componentCfg, err := handler.FromDocumentMap(docmap)
		if err != nil {
			return err
		}

		if componentCfg != nil {
			clusterCfg.ComponentConfigs[handler.GroupVersion.Group] = componentCfg
		}
	}

	return nil
}


// FetchFromClusterWithLocalOverwrites fetches component configs from a cluster and overwrites them locally with
// the ones present in the supplied document map. If any UnsupportedConfigVersionError are not handled by the configs
// in the document map, the function returns them all as a single UnsupportedConfigVersionsErrorMap.
// This function is normally called only in some specific cases during upgrade.
func FetchFromClusterWithLocalOverwrites(clusterCfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, docmap kubeadmapi.DocumentMap) error {
	ensureInitializedComponentConfigs(clusterCfg)

	oldVersionErrs := UnsupportedConfigVersionsErrorMap{}

	for _, handler := range known {
		componentCfg, err := handler.FromCluster(client, clusterCfg)
		if err != nil {
			if vererr, ok := err.(*UnsupportedConfigVersionError); ok {
				oldVersionErrs[handler.GroupVersion.Group] = vererr
			} else {
				return err
			}
		} else if componentCfg != nil {
			clusterCfg.ComponentConfigs[handler.GroupVersion.Group] = componentCfg
		}
	}

	for _, handler := range known {
		componentCfg, err := handler.FromDocumentMap(docmap)
		if err != nil {
			if vererr, ok := err.(*UnsupportedConfigVersionError); ok {
				oldVersionErrs[handler.GroupVersion.Group] = vererr
			} else {
				return err
			}
		} else if componentCfg != nil {
			clusterCfg.ComponentConfigs[handler.GroupVersion.Group] = componentCfg
			delete(oldVersionErrs, handler.GroupVersion.Group)
		}
	}

	if len(oldVersionErrs) != 0 {
		return oldVersionErrs
	}

	return nil
}