// ValidateCertPeriod checks if the certificate is valid relative to the current time
// (+/- offset)
func ValidateCertPeriod(cert *x509.Certificate, offset time.Duration) error {
	period := fmt.Sprintf("NotBefore: %v, NotAfter: %v", cert.NotBefore, cert.NotAfter)
	now := time.Now().Add(offset)
	if now.Before(cert.NotBefore) {
		return errors.Errorf("the certificate is not valid yet: %s", period)
	}
	if now.After(cert.NotAfter) {
		return errors.Errorf("the certificate has expired: %s", period)
	}
	return nil
}


// VerifyCertChain verifies that a certificate has a valid chain of
// intermediate CAs back to the root CA
func VerifyCertChain(cert *x509.Certificate, intermediates []*x509.Certificate, root *x509.Certificate) error {
	rootPool := x509.NewCertPool()
	rootPool.AddCert(root)

	intermediatePool := x509.NewCertPool()
	for _, c := range intermediates {
		intermediatePool.AddCert(c)
	}

	verifyOptions := x509.VerifyOptions{
		Roots:         rootPool,
		Intermediates: intermediatePool,
		KeyUsages:     []x509.ExtKeyUsage{x509.ExtKeyUsageAny},
	}

	if _, err := cert.Verify(verifyOptions); err != nil {
		return err
	}

	return nil
}


func NewSet() *Set

// NewSet returns a new, empty PubKeyPinSet
func NewSet() *Set {
	return &Set{make(map[string]bool)}
}


// Allow adds an allowed public key hash to the Set
func (s *Set) Allow(pubKeyHashes ...string) error {
	for _, pubKeyHash := range pubKeyHashes {
		parts := strings.Split(pubKeyHash, ":")
		if len(parts) != 2 {
			return errors.Errorf("invalid hash, expected \"format:hex-value\". "+
				"Known format(s) are: %s", supportedFormats)
		}
		format, value := parts[0], parts[1]

		switch strings.ToLower(format) {
		case "sha256":
			if err := s.allowSHA256(value); err != nil {
				return errors.Errorf("invalid hash %q, %v", pubKeyHash, err)
			}
		default:
			return errors.Errorf("unknown hash format %q. Known format(s) are: %s", format, supportedFormats)
		}
	}
	return nil
}


// CheckAny checks if at least one certificate matches one of the public keys in the set
func (s *Set) CheckAny(certificates []*x509.Certificate) error {
	var hashes []string

	for _, certificate := range certificates {
		if s.checkSHA256(certificate) {
			return nil
		}

		hashes = append(hashes, Hash(certificate))
	}
	return errors.Errorf("none of the public keys %q are pinned", strings.Join(hashes, ":"))
}


func (s *Set) Empty() bool

// Empty returns true if the Set contains no pinned public keys.
func (s *Set) Empty() bool {
	return len(s.sha256Hashes) == 0
}


// Hash calculates the SHA-256 hash of the Subject Public Key Information (SPKI)
// object in an x509 certificate (in DER encoding). It returns the full hash as a
// hex encoded string (suitable for passing to Set.Allow).
func Hash(certificate *x509.Certificate) string {
	spkiHash := sha256.Sum256(certificate.RawSubjectPublicKeyInfo)
	return formatSHA256 + ":" + strings.ToLower(hex.EncodeToString(spkiHash[:]))
}


// allowSHA256 validates a "sha256" format hash and adds a canonical version of it into the Set
func (s *Set) allowSHA256(hash string) error {
	// validate that the hash is the right length to be a full SHA-256 hash
	hashLength := hex.DecodedLen(len(hash))
	if hashLength != sha256.Size {
		return errors.Errorf("expected a %d byte SHA-256 hash, found %d bytes", sha256.Size, hashLength)
	}

	// validate that the hash is valid hex
	_, err := hex.DecodeString(hash)
	if err != nil {
		return errors.Wrap(err, "could not decode SHA-256 from hex")
	}

	// in the end, just store the original hex string in memory (in lowercase)
	s.sha256Hashes[strings.ToLower(hash)] = true
	return nil
}


// checkSHA256 returns true if the certificate's "sha256" hash is pinned in the Set
func (s *Set) checkSHA256(certificate *x509.Certificate) bool {
	actualHash := sha256.Sum256(certificate.RawSubjectPublicKeyInfo)
	actualHashHex := strings.ToLower(hex.EncodeToString(actualHash[:]))
	return s.sha256Hashes[actualHashHex]
}


// NewContainerRuntime sets up and returns a ContainerRuntime struct
func NewContainerRuntime(execer utilsexec.Interface, criSocket string) (ContainerRuntime, error) {
	const toolName = "crictl"
	crictlPath, err := execer.LookPath(toolName)
	if err != nil {
		return nil, errors.Wrapf(err, "%s is required by the container runtime", toolName)
	}
	return &CRIRuntime{execer, criSocket, crictlPath}, nil
}


// Socket returns the CRI socket endpoint
func (runtime *CRIRuntime) Socket() string {
	return runtime.criSocket
}


// crictl creates a crictl command for the provided args.
func (runtime *CRIRuntime) crictl(args ...string) utilsexec.Cmd {
	cmd := runtime.exec.Command(runtime.crictlPath, append([]string{"-r", runtime.Socket()}, args...)...)
	cmd.SetEnv(os.Environ())
	return cmd
}


// IsRunning checks if runtime is running
func (runtime *CRIRuntime) IsRunning() error {
	if out, err := runtime.crictl("info").CombinedOutput(); err != nil {
		return errors.Wrapf(err, "container runtime is not running: output: %s, error", string(out))
	}
	return nil
}


// ListKubeContainers lists running k8s CRI pods
func (runtime *CRIRuntime) ListKubeContainers() ([]string, error) {
	// Disable debug mode regardless how the crictl is configured so that the debug info won't be
	// iterpreted to the Pod ID.
	args := []string{"-D=false", "pods", "-q"}
	out, err := runtime.crictl(args...).CombinedOutput()
	if err != nil {
		return nil, errors.Wrapf(err, "output: %s, error", string(out))
	}
	pods := []string{}
	pods = append(pods, strings.Fields(string(out))...)
	return pods, nil
}


// RemoveContainers removes running k8s pods
func (runtime *CRIRuntime) RemoveContainers(containers []string) error {
	errs := []error{}
	for _, container := range containers {
		var lastErr error
		for i := 0; i < constants.RemoveContainerRetry; i++ {
			klog.V(5).Infof("Attempting to remove container %v", container)
			out, err := runtime.crictl("stopp", container).CombinedOutput()
			if err != nil {
				lastErr = errors.Wrapf(err, "failed to stop running pod %s: output: %s", container, string(out))
				continue
			}
			out, err = runtime.crictl("rmp", container).CombinedOutput()
			if err != nil {
				lastErr = errors.Wrapf(err, "failed to remove running container %s: output: %s", container, string(out))
				continue
			}
			lastErr = nil
			break
		}

		if lastErr != nil {
			errs = append(errs, lastErr)
		}
	}
	return errorsutil.NewAggregate(errs)
}


// PullImage pulls the image
func (runtime *CRIRuntime) PullImage(image string) error {
	var err error
	var out []byte
	for i := 0; i < constants.PullImageRetry; i++ {
		out, err = runtime.crictl("pull", image).CombinedOutput()
		if err == nil {
			return nil
		}
	}
	return errors.Wrapf(err, "output: %s, error", out)
}


// ImageExists checks to see if the image exists on the system
func (runtime *CRIRuntime) ImageExists(image string) (bool, error) {
	err := runtime.crictl("inspecti", image).Run()
	return err == nil, nil
}


// detectCRISocketImpl is separated out only for test purposes, DON'T call it directly, use DetectCRISocket instead
func detectCRISocketImpl(isSocket func(string) bool, knownCRISockets []string) (string, error) {
	foundCRISockets := []string{}

	for _, socket := range knownCRISockets {
		if isSocket(socket) {
			foundCRISockets = append(foundCRISockets, socket)
		}
	}

	switch len(foundCRISockets) {
	case 0:
		// Fall back to the default socket if no CRI is detected, we can error out later on if we need it
		return constants.DefaultCRISocket, nil
	case 1:
		// Precisely one CRI found, use that
		return foundCRISockets[0], nil
	default:
		// Multiple CRIs installed?
		return "", errors.Errorf("Found multiple CRI endpoints on the host. Please define which one do you wish "+
			"to use by setting the 'criSocket' field in the kubeadm configuration file: %s",
			strings.Join(foundCRISockets, ", "))
	}
}


func DetectCRISocket() (string, error)

// DetectCRISocket uses a list of known CRI sockets to detect one. If more than one or none is discovered, an error is returned.
func DetectCRISocket() (string, error) {
	return detectCRISocketImpl(isExistingSocket, defaultKnownCRISockets)
}


// SandboxImage returns the sandbox image used by the container runtime
func (runtime *CRIRuntime) SandboxImage() (string, error) {
	args := []string{"-D=false", "info", "-o", "go-template", "--template", "{{.config.sandboxImage}}"}
	out, err := runtime.crictl(args...).CombinedOutput()
	if err != nil {
		return "", errors.Wrapf(err, "output: %s, error", string(out))
	}

	sandboxImage := strings.TrimSpace(string(out))
	if len(sandboxImage) > 0 {
		return sandboxImage, nil
	}

	return "", errors.Errorf("the detected sandbox image is empty")
}


func isExistingSocket(path string) bool

// isExistingSocket checks if path exists and is domain socket
func isExistingSocket(path string) bool {
	u, err := url.Parse(path)
	if err != nil {
		// should not happen, since we are trying to access known / hardcoded sockets
		return false
	}

	c, err := net.Dial(u.Scheme, u.Path)
	if err != nil {
		return false
	}
	defer c.Close()
	return true
}


// isExistingSocket checks if path exists and is domain socket
func isExistingSocket(path string) bool {
	u, err := url.Parse(path)
	if err != nil {
		// should not happen, since we are trying to access known / hardcoded sockets
		return false
	}

	// the dial path must be without "npipe://"
	_, err = winio.DialPipe(u.Path, nil)
	if err != nil {
		return false
	}

	return true
}


// ComponentPod returns a Pod object from the container, volume and annotations specifications
func ComponentPod(container v1.Container, volumes map[string]v1.Volume, annotations map[string]string) v1.Pod {
	// priority value for system-node-critical class
	priority := int32(2000001000)
	return v1.Pod{
		TypeMeta: metav1.TypeMeta{
			APIVersion: "v1",
			Kind:       "Pod",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      container.Name,
			Namespace: metav1.NamespaceSystem,
			// The component and tier labels are useful for quickly identifying the control plane Pods when doing a .List()
			// against Pods in the kube-system namespace. Can for example be used together with the WaitForPodsWithLabel function
			Labels:      map[string]string{"component": container.Name, "tier": kubeadmconstants.ControlPlaneTier},
			Annotations: annotations,
		},
		Spec: v1.PodSpec{
			Containers:        []v1.Container{container},
			Priority:          &priority,
			PriorityClassName: "system-node-critical",
			HostNetwork:       true,
			Volumes:           VolumeMapToSlice(volumes),
			SecurityContext: &v1.PodSecurityContext{
				SeccompProfile: &v1.SeccompProfile{
					Type: v1.SeccompProfileTypeRuntimeDefault,
				},
			},
		},
	}
}


// ComponentResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU
func ComponentResources(cpu string) v1.ResourceRequirements {
	return v1.ResourceRequirements{
		Requests: v1.ResourceList{
			v1.ResourceName(v1.ResourceCPU): resource.MustParse(cpu),
		},
	}
}


// NewVolume creates a v1.Volume with a hostPath mount to the specified location
func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume {
	return v1.Volume{
		Name: name,
		VolumeSource: v1.VolumeSource{
			HostPath: &v1.HostPathVolumeSource{
				Path: path,
				Type: pathType,
			},
		},
	}
}


// NewVolumeMount creates a v1.VolumeMount to the specified location
func NewVolumeMount(name, path string, readOnly bool) v1.VolumeMount {
	return v1.VolumeMount{
		Name:      name,
		MountPath: path,
		ReadOnly:  readOnly,
	}
}


// VolumeMapToSlice returns a slice of volumes from a map's values
func VolumeMapToSlice(volumes map[string]v1.Volume) []v1.Volume {
	v := make([]v1.Volume, 0, len(volumes))

	for _, vol := range volumes {
		v = append(v, vol)
	}

	sort.Slice(v, func(i, j int) bool {
		return strings.Compare(v[i].Name, v[j].Name) == -1
	})

	return v
}


// VolumeMountMapToSlice returns a slice of volumes from a map's values
func VolumeMountMapToSlice(volumeMounts map[string]v1.VolumeMount) []v1.VolumeMount {
	v := make([]v1.VolumeMount, 0, len(volumeMounts))

	for _, volMount := range volumeMounts {
		v = append(v, volMount)
	}

	sort.Slice(v, func(i, j int) bool {
		return strings.Compare(v[i].Name, v[j].Name) == -1
	})

	return v
}


// GetExtraParameters builds a list of flag arguments two string-string maps, one with default, base commands and one with overrides
func GetExtraParameters(overrides map[string]string, defaults map[string]string) []string {
	var command []string
	for k, v := range overrides {
		if len(v) > 0 {
			command = append(command, fmt.Sprintf("--%s=%s", k, v))
		}
	}
	for k, v := range defaults {
		if _, overrideExists := overrides[k]; !overrideExists {
			command = append(command, fmt.Sprintf("--%s=%s", k, v))
		}
	}
	return command
}


// PatchStaticPod applies patches stored in patchesDir to a static Pod.
func PatchStaticPod(pod *v1.Pod, patchesDir string, output io.Writer) (*v1.Pod, error) {
	// Marshal the Pod manifest into YAML.
	podYAML, err := kubeadmutil.MarshalToYaml(pod, v1.SchemeGroupVersion)
	if err != nil {
		return pod, errors.Wrapf(err, "failed to marshal Pod manifest to YAML")
	}

	patchManager, err := patches.GetPatchManagerForPath(patchesDir, patches.KnownTargets(), output)
	if err != nil {
		return pod, err
	}

	patchTarget := &patches.PatchTarget{
		Name:                      pod.Name,
		StrategicMergePatchObject: v1.Pod{},
		Data:                      podYAML,
	}
	if err := patchManager.ApplyPatchesToTarget(patchTarget); err != nil {
		return pod, err
	}

	obj, err := kubeadmutil.UnmarshalFromYaml(patchTarget.Data, v1.SchemeGroupVersion)
	if err != nil {
		return pod, errors.Wrap(err, "failed to unmarshal patched manifest from YAML")
	}

	pod2, ok := obj.(*v1.Pod)
	if !ok {
		return pod, errors.Wrap(err, "patched manifest is not a valid Pod object")
	}

	return pod2, nil
}


// WriteStaticPodToDisk writes a static pod file to disk
func WriteStaticPodToDisk(componentName, manifestDir string, pod v1.Pod) error {

	// creates target folder if not already exists
	if err := os.MkdirAll(manifestDir, 0700); err != nil {
		return errors.Wrapf(err, "failed to create directory %q", manifestDir)
	}

	// writes the pod to disk
	serialized, err := kubeadmutil.MarshalToYaml(&pod, v1.SchemeGroupVersion)
	if err != nil {
		return errors.Wrapf(err, "failed to marshal manifest for %q to YAML", componentName)
	}

	filename := kubeadmconstants.GetStaticPodFilepath(componentName, manifestDir)

	if err := os.WriteFile(filename, serialized, 0600); err != nil {
		return errors.Wrapf(err, "failed to write static pod manifest file for %q (%q)", componentName, filename)
	}

	return nil
}


// ReadStaticPodFromDisk reads a static pod file from disk
func ReadStaticPodFromDisk(manifestPath string) (*v1.Pod, error) {
	buf, err := os.ReadFile(manifestPath)
	if err != nil {
		return &v1.Pod{}, errors.Wrapf(err, "failed to read manifest for %q", manifestPath)
	}

	obj, err := kubeadmutil.UnmarshalFromYaml(buf, v1.SchemeGroupVersion)
	if err != nil {
		return &v1.Pod{}, errors.Errorf("failed to unmarshal manifest for %q from YAML: %v", manifestPath, err)
	}

	pod := obj.(*v1.Pod)

	return pod, nil
}


// LivenessProbe creates a Probe object with a HTTPGet handler
func LivenessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe {
	// sets initialDelaySeconds same as periodSeconds to skip one period before running a check
	return createHTTPProbe(host, path, port, scheme, 10, 15, 8, 10)
}


// ReadinessProbe creates a Probe object with a HTTPGet handler
func ReadinessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe {
	// sets initialDelaySeconds as '0' because we don't want to delay user infrastructure checks
	// looking for "ready" status on kubeadm static Pods
	return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 1)
}


// StartupProbe creates a Probe object with a HTTPGet handler
func StartupProbe(host, path string, port int, scheme v1.URIScheme, timeoutForControlPlane *metav1.Duration) *v1.Probe {
	periodSeconds, timeoutForControlPlaneSeconds := int32(10), kubeadmconstants.DefaultControlPlaneTimeout.Seconds()
	if timeoutForControlPlane != nil {
		timeoutForControlPlaneSeconds = timeoutForControlPlane.Seconds()
	}
	// sets failureThreshold big enough to guarantee the full timeout can cover the worst case scenario for the control-plane to come alive
	// we ignore initialDelaySeconds in the calculation here for simplicity
	failureThreshold := int32(math.Ceil(timeoutForControlPlaneSeconds / float64(periodSeconds)))
	// sets initialDelaySeconds same as periodSeconds to skip one period before running a check
	return createHTTPProbe(host, path, port, scheme, periodSeconds, 15, failureThreshold, periodSeconds)
}


func createHTTPProbe(host, path string, port int, scheme v1.URIScheme, initialDelaySeconds, timeoutSeconds, failureThreshold, periodSeconds int32) *v1.Probe {
	return &v1.Probe{
		ProbeHandler: v1.ProbeHandler{
			HTTPGet: &v1.HTTPGetAction{
				Host:   host,
				Path:   path,
				Port:   intstr.FromInt(port),
				Scheme: scheme,
			},
		},
		InitialDelaySeconds: initialDelaySeconds,
		TimeoutSeconds:      timeoutSeconds,
		FailureThreshold:    failureThreshold,
		PeriodSeconds:       periodSeconds,
	}
}


// GetAPIServerProbeAddress returns the probe address for the API server
func GetAPIServerProbeAddress(endpoint *kubeadmapi.APIEndpoint) string {
	if endpoint != nil && endpoint.AdvertiseAddress != "" {
		return getProbeAddress(endpoint.AdvertiseAddress)
	}

	return "127.0.0.1"
}


// GetControllerManagerProbeAddress returns the kubernetes controller manager probe address
func GetControllerManagerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) string {
	if addr, exists := cfg.ControllerManager.ExtraArgs[kubeControllerManagerBindAddressArg]; exists {
		return getProbeAddress(addr)
	}
	return "127.0.0.1"
}


// GetSchedulerProbeAddress returns the kubernetes scheduler probe address
func GetSchedulerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) string {
	if addr, exists := cfg.Scheduler.ExtraArgs[kubeSchedulerBindAddressArg]; exists {
		return getProbeAddress(addr)
	}
	return "127.0.0.1"
}


// GetEtcdProbeEndpoint takes a kubeadm Etcd configuration object and attempts to parse
// the first URL in the listen-metrics-urls argument, returning an etcd probe hostname,
// port and scheme
func GetEtcdProbeEndpoint(cfg *kubeadmapi.Etcd, isIPv6 bool) (string, int, v1.URIScheme) {
	localhost := "127.0.0.1"
	if isIPv6 {
		localhost = "::1"
	}
	if cfg.Local == nil || cfg.Local.ExtraArgs == nil {
		return localhost, kubeadmconstants.EtcdMetricsPort, v1.URISchemeHTTP
	}
	if arg, exists := cfg.Local.ExtraArgs["listen-metrics-urls"]; exists {
		// Use the first url in the listen-metrics-urls if multiple URL's are specified.
		arg = strings.Split(arg, ",")[0]
		parsedURL, err := url.Parse(arg)
		if err != nil {
			return localhost, kubeadmconstants.EtcdMetricsPort, v1.URISchemeHTTP
		}
		// Parse scheme
		scheme := v1.URISchemeHTTP
		if parsedURL.Scheme == "https" {
			scheme = v1.URISchemeHTTPS
		}
		// Parse hostname
		hostname := parsedURL.Hostname()
		if len(hostname) == 0 {
			hostname = localhost
		}
		// Parse port
		port := kubeadmconstants.EtcdMetricsPort
		portStr := parsedURL.Port()
		if len(portStr) != 0 {
			p, err := kubeadmutil.ParsePort(portStr)
			if err == nil {
				port = p
			}
		}
		return hostname, port, scheme
	}
	return localhost, kubeadmconstants.EtcdMetricsPort, v1.URISchemeHTTP
}


// ManifestFilesAreEqual compares 2 files. It returns true if their contents are equal, false otherwise
func ManifestFilesAreEqual(path1, path2 string) (bool, error) {
	pod1, err := ReadStaticPodFromDisk(path1)
	if err != nil {
		return false, err
	}
	pod2, err := ReadStaticPodFromDisk(path2)
	if err != nil {
		return false, err
	}

	hasher := md5.New()
	DeepHashObject(hasher, pod1)
	hash1 := hasher.Sum(nil)[0:]
	DeepHashObject(hasher, pod2)
	hash2 := hasher.Sum(nil)[0:]

	return bytes.Equal(hash1, hash2), nil
}


// getProbeAddress returns a valid probe address.
// Kubeadm uses the bind-address to configure the probe address. It's common to use the
// unspecified address "0.0.0.0" or "::" as bind-address when we want to listen in all interfaces,
// however this address can't be used as probe #86504.
// If the address is an unspecified address getProbeAddress returns empty,
// that means that kubelet will use the PodIP as probe address.
func getProbeAddress(addr string) string {
	if addr == "0.0.0.0" || addr == "::" {
		return ""
	}
	return addr
}


func GetUsersAndGroups() (*users.UsersAndGroups, error) {
	var err error
	usersAndGroupsOnce.Do(func() {
		usersAndGroups, err = users.AddUsersAndGroups()
	})
	return usersAndGroups, err
}


// DeepHashObject writes specified object to hash using the spew library
// which follows pointers and prints actual values of the nested objects
// ensuring the hash does not change when a pointer changes.
// Copied from k8s.io/kubernetes/pkg/util/hash/hash.go#DeepHashObject
func DeepHashObject(hasher hash.Hash, objectToWrite interface{}) {
	hasher.Reset()
	printer := spew.ConfigState{
		Indent:         " ",
		SortKeys:       true,
		DisableMethods: true,
		SpewKeys:       true,
	}
	printer.Fprintf(hasher, "%#v", objectToWrite)
}


// RunComponentAsNonRoot updates the pod manifest and the hostVolume permissions to run as non root.
func RunComponentAsNonRoot(componentName string, pod *v1.Pod, usersAndGroups *users.UsersAndGroups, cfg *kubeadmapi.ClusterConfiguration) error {
	switch componentName {
	case kubeadmconstants.KubeAPIServer:
		return runKubeAPIServerAsNonRoot(
			pod,
			usersAndGroups.Users.ID(kubeadmconstants.KubeAPIServerUserName),
			usersAndGroups.Groups.ID(kubeadmconstants.KubeAPIServerUserName),
			usersAndGroups.Groups.ID(kubeadmconstants.ServiceAccountKeyReadersGroupName),
			users.UpdatePathOwnerAndPermissions,
			cfg,
		)
	case kubeadmconstants.KubeControllerManager:
		return runKubeControllerManagerAsNonRoot(
			pod,
			usersAndGroups.Users.ID(kubeadmconstants.KubeControllerManagerUserName),
			usersAndGroups.Groups.ID(kubeadmconstants.KubeControllerManagerUserName),
			usersAndGroups.Groups.ID(kubeadmconstants.ServiceAccountKeyReadersGroupName),
			users.UpdatePathOwnerAndPermissions,
			cfg,
		)
	case kubeadmconstants.KubeScheduler:
		return runKubeSchedulerAsNonRoot(
			pod,
			usersAndGroups.Users.ID(kubeadmconstants.KubeSchedulerUserName),
			usersAndGroups.Groups.ID(kubeadmconstants.KubeSchedulerUserName),
			users.UpdatePathOwnerAndPermissions,
		)
	case kubeadmconstants.Etcd:
		return runEtcdAsNonRoot(
			pod,
			usersAndGroups.Users.ID(kubeadmconstants.EtcdUserName),
			usersAndGroups.Groups.ID(kubeadmconstants.EtcdUserName),
			users.UpdatePathOwnerAndPermissions,
			users.UpdatePathOwner,
			cfg,
		)
	}
	return errors.New(fmt.Sprintf("component name %q is not valid", componentName))
}


// runKubeAPIServerAsNonRoot updates the pod manifest and the hostVolume permissions to run kube-apiserver as non root.
func runKubeAPIServerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error {
	saPublicKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName)
	if err := updatePathOwnerAndPermissions(saPublicKeyFile, *runAsUser, *runAsGroup, 0600); err != nil {
		return err
	}
	saPrivateKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPrivateKeyName)
	if err := updatePathOwnerAndPermissions(saPrivateKeyFile, 0, *supplementalGroup, 0640); err != nil {
		return err
	}
	apiServerKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKeyName)
	if err := updatePathOwnerAndPermissions(apiServerKeyFile, *runAsUser, *runAsGroup, 0600); err != nil {
		return err
	}
	apiServerKubeletClientKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientKeyName)
	if err := updatePathOwnerAndPermissions(apiServerKubeletClientKeyFile, *runAsUser, *runAsGroup, 0600); err != nil {
		return err
	}
	frontProxyClientKeyName := filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName)
	if err := updatePathOwnerAndPermissions(frontProxyClientKeyName, *runAsUser, *runAsGroup, 0600); err != nil {
		return err
	}
	if cfg.Etcd.External == nil {
		apiServerEtcdClientKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientKeyName)
		if err := updatePathOwnerAndPermissions(apiServerEtcdClientKeyFile, *runAsUser, *runAsGroup, 0600); err != nil {
			return err
		}
	}
	pod.Spec.Containers[0].SecurityContext = &v1.SecurityContext{
		Capabilities: &v1.Capabilities{
			// We drop all capabilities that are added by default.
			Drop: []v1.Capability{"ALL"},
			// kube-apiserver binary has the file capability cap_net_bind_service applied to it.
			// This means that we must add this capability when running as non-root even if the
			// capability is not required.
			Add: []v1.Capability{"NET_BIND_SERVICE"},
		},
	}
	pod.Spec.SecurityContext.RunAsGroup = runAsGroup
	pod.Spec.SecurityContext.RunAsUser = runAsUser
	pod.Spec.SecurityContext.SupplementalGroups = []int64{*supplementalGroup}
	return nil
}


// runKubeControllerManagerAsNonRoot updates the pod manifest and the hostVolume permissions to run kube-controller-manager as non root.
func runKubeControllerManagerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error {
	kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName)
	if err := updatePathOwnerAndPermissions(kubeconfigFile, *runAsUser, *runAsGroup, 0600); err != nil {
		return err
	}
	saPrivateKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPrivateKeyName)
	if err := updatePathOwnerAndPermissions(saPrivateKeyFile, 0, *supplementalGroup, 0640); err != nil {
		return err
	}
	if res, _ := certphase.UsingExternalCA(cfg); !res {
		caKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.CAKeyName)
		err := updatePathOwnerAndPermissions(caKeyFile, *runAsUser, *runAsGroup, 0600)
		if err != nil {
			return err
		}
	}
	pod.Spec.Containers[0].SecurityContext = &v1.SecurityContext{
		AllowPrivilegeEscalation: pointer.Bool(false),
		Capabilities: &v1.Capabilities{
			// We drop all capabilities that are added by default.
			Drop: []v1.Capability{"ALL"},
		},
	}
	pod.Spec.SecurityContext.RunAsUser = runAsUser
	pod.Spec.SecurityContext.RunAsGroup = runAsGroup
	pod.Spec.SecurityContext.SupplementalGroups = []int64{*supplementalGroup}
	return nil
}


// runKubeSchedulerAsNonRoot updates the pod manifest and the hostVolume permissions to run kube-scheduler as non root.
func runKubeSchedulerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc) error {
	kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName)
	if err := updatePathOwnerAndPermissions(kubeconfigFile, *runAsUser, *runAsGroup, 0600); err != nil {
		return err
	}
	pod.Spec.Containers[0].SecurityContext = &v1.SecurityContext{
		AllowPrivilegeEscalation: pointer.Bool(false),
		// We drop all capabilities that are added by default.
		Capabilities: &v1.Capabilities{
			Drop: []v1.Capability{"ALL"},
		},
	}
	pod.Spec.SecurityContext.RunAsUser = runAsUser
	pod.Spec.SecurityContext.RunAsGroup = runAsGroup
	return nil
}


// runEtcdAsNonRoot updates the pod manifest and the hostVolume permissions to run etcd as non root.
func runEtcdAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc, updatePathOwner pathOwnerUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error {
	if err := updatePathOwner(cfg.Etcd.Local.DataDir, *runAsUser, *runAsGroup); err != nil {
		return err
	}
	etcdServerKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName)
	if err := updatePathOwnerAndPermissions(etcdServerKeyFile, *runAsUser, *runAsGroup, 0600); err != nil {
		return err
	}
	etcdPeerKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName)
	if err := updatePathOwnerAndPermissions(etcdPeerKeyFile, *runAsUser, *runAsGroup, 0600); err != nil {
		return err
	}
	pod.Spec.Containers[0].SecurityContext = &v1.SecurityContext{
		AllowPrivilegeEscalation: pointer.Bool(false),
		// We drop all capabilities that are added by default.
		Capabilities: &v1.Capabilities{
			Drop: []v1.Capability{"ALL"},
		},
	}
	pod.Spec.SecurityContext.RunAsUser = runAsUser
	pod.Spec.SecurityContext.RunAsGroup = runAsGroup
	return nil
}


// RunComponentAsNonRoot is a NO-OP on non linux.
func RunComponentAsNonRoot(componentName string, pod *v1.Pod, usersAndGroups *users.UsersAndGroups, cfg *kubeadmapi.ClusterConfiguration) error {
	return nil
}


// ParseTemplate validates and parses passed as argument template
func ParseTemplate(strtmpl string, obj interface{}) ([]byte, error) {
	var buf bytes.Buffer
	tmpl, err := template.New("template").Parse(strtmpl)
	if err != nil {
		return nil, errors.Wrap(err, "error when parsing template")
	}
	err = tmpl.Execute(&buf, obj)
	if err != nil {
		return nil, errors.Wrap(err, "error when executing template")
	}
	return buf.Bytes(), nil
}


// ID returns the ID for an entry based on the entry name.
// In case of a user entry it returns the user UID.
// In case of a group entry it returns the group GID.
// It returns nil if no such entry exists.
func (u *EntryMap) ID(name string) *int64 {
	entry, ok := u.entries[name]
	if !ok {
		return nil
	}
	id := entry.id
	return &id
}


func (u *EntryMap) String() string

// String converts an EntryMap object to a readable string.
func (u *EntryMap) String() string {
	lines := make([]string, 0, len(u.entries))
	for k, e := range u.entries {
		lines = append(lines, fmt.Sprintf("%s{%d,%d};", k, e.id, e.gid))
	}
	sort.Strings(lines)
	return strings.Join(lines, "")
}


// Is a public wrapper around addUsersAndGroupsImpl with default system file paths.
func AddUsersAndGroups() (*UsersAndGroups, error) {
	return addUsersAndGroupsImpl(fileEtcLoginDefs, fileEtcPasswd, fileEtcGroup)
}


// addUsersAndGroupsImpl adds the managed users and groups to the files specified
// by pathUsers and pathGroups. It uses the file specified with pathLoginDef to
// determine limits for UID and GID. If managed users and groups exist in these files
// validation is performed on them. The function returns a pointer to a Users object
// that can be used to return UID and GID of managed users.
func addUsersAndGroupsImpl(pathLoginDef, pathUsers, pathGroups string) (*UsersAndGroups, error) {
	klog.V(1).Info("Adding managed users and groups")
	klog.V(klogLevel).Infof("Parsing %q", pathLoginDef)

	// Read and parse /etc/login.def. Some distributions might be missing this file, which makes
	// them non-standard. If an error occurs fallback to defaults by passing an empty string
	// to parseLoginDefs().
	var loginDef string
	f, close, err := openFileWithLock(pathLoginDef)
	if err != nil {
		klog.V(1).Infof("Could not open %q, using default system limits: %v", pathLoginDef, err)
	} else {
		loginDef, err = readFile(f)
		if err != nil {
			klog.V(1).Infof("Could not read %q, using default system limits: %v", pathLoginDef, err)
		}
		close()
	}
	limits, err := parseLoginDefs(loginDef)
	if err != nil {
		return nil, err
	}

	klog.V(klogLevel).Infof("Using system UID/GID limits: %+v", limits)
	klog.V(klogLevel).Infof("Parsing %q and %q", pathUsers, pathGroups)

	// Open /etc/passwd and /etc/group with locks.
	fUsers, close, err := openFileWithLock(pathUsers)
	if err != nil {
		return nil, err
	}
	defer close()
	fGroups, close, err := openFileWithLock(pathGroups)
	if err != nil {
		return nil, err
	}
	defer close()

	// Read the files.
	fileUsers, err := readFile(fUsers)
	if err != nil {
		return nil, err
	}
	fileGroups, err := readFile(fGroups)
	if err != nil {
		return nil, err
	}

	// Parse the files.
	users, err := parseEntries(fileUsers, totalFieldsUser)
	if err != nil {
		return nil, errors.Wrapf(err, "could not parse %q", pathUsers)
	}
	groups, err := parseEntries(fileGroups, totalFieldsGroup)
	if err != nil {
		return nil, errors.Wrapf(err, "could not parse %q", pathGroups)
	}

	klog.V(klogLevel).Info("Validating existing users and groups")

	// Validate for existing tracked entries based on limits.
	usersToCreate, groupsToCreate, err := validateEntries(users, groups, limits)
	if err != nil {
		return nil, errors.Wrap(err, "error validating existing users and groups")
	}

	// Allocate and assign IDs to users / groups.
	allocUIDs, err := allocateIDs(users, limits.minUID, limits.maxUID, len(usersToCreate))
	if err != nil {
		return nil, err
	}
	allocGIDs, err := allocateIDs(groups, limits.minGID, limits.maxGID, len(groupsToCreate))
	if err != nil {
		return nil, err
	}
	if err := assignUserAndGroupIDs(groups, usersToCreate, groupsToCreate, allocUIDs, allocGIDs); err != nil {
		return nil, err
	}

	if len(usersToCreate) > 0 {
		klog.V(klogLevel).Infof("Adding users: %s", entriesToString(usersToCreate))
	}
	if len(groupsToCreate) > 0 {
		klog.V(klogLevel).Infof("Adding groups: %s", entriesToString(groupsToCreate))
	}

	// Add users and groups.
	fileUsers = addEntries(fileUsers, usersToCreate, createUser)
	fileGroups = addEntries(fileGroups, groupsToCreate, createGroup)

	// Write the files.
	klog.V(klogLevel).Infof("Writing %q and %q", pathUsers, pathGroups)
	if err := writeFile(fUsers, fileUsers); err != nil {
		return nil, err
	}
	if err := writeFile(fGroups, fileGroups); err != nil {
		return nil, err
	}

	// Prepare the maps of users and groups.
	usersConcat := append(users, usersToCreate...)
	mapUsers, err := entriesToEntryMap(usersConcat, usersToCreateSpec)
	if err != nil {
		return nil, err
	}
	groupsConcat := append(groups, groupsToCreate...)
	mapGroups, err := entriesToEntryMap(groupsConcat, groupsToCreateSpec)
	if err != nil {
		return nil, err
	}
	return &UsersAndGroups{Users: mapUsers, Groups: mapGroups}, nil
}


func RemoveUsersAndGroups() error

// RemoveUsersAndGroups is a public wrapper around removeUsersAndGroupsImpl with
// default system file paths.
func RemoveUsersAndGroups() error {
	return removeUsersAndGroupsImpl(fileEtcPasswd, fileEtcGroup)
}


// removeUsersAndGroupsImpl removes the managed users and groups from the files specified
// by pathUsers and pathGroups.
func removeUsersAndGroupsImpl(pathUsers, pathGroups string) error {
	klog.V(1).Info("Removing managed users and groups")
	klog.V(klogLevel).Infof("Opening %q and %q", pathUsers, pathGroups)

	// Open /etc/passwd and /etc/group.
	fUsers, close, err := openFileWithLock(pathUsers)
	if err != nil {
		return err
	}
	defer close()
	fGroups, close, err := openFileWithLock(pathGroups)
	if err != nil {
		return err
	}
	defer close()

	// Read the files.
	fileUsers, err := readFile(fUsers)
	if err != nil {
		return err
	}
	fileGroups, err := readFile(fGroups)
	if err != nil {
		return err
	}

	klog.V(klogLevel).Infof("Removing users: %s", entriesToString(usersToCreateSpec))
	klog.V(klogLevel).Infof("Removing groups: %s", entriesToString(groupsToCreateSpec))

	// Delete users / groups.
	fileUsers, _ = removeEntries(fileUsers, usersToCreateSpec)
	fileGroups, _ = removeEntries(fileGroups, groupsToCreateSpec)

	klog.V(klogLevel).Infof("Writing %q and %q", pathUsers, pathGroups)

	// Write the files.
	if err := writeFile(fUsers, fileUsers); err != nil {
		return err
	}
	if err := writeFile(fGroups, fileGroups); err != nil {
		return err
	}

	return nil
}


// parseLoginDefs can be used to parse an /etc/login.defs file and obtain system ranges for UID and GID.
// Passing an empty string will return the defaults. The defaults are 100-999 for both UID and GID.
func parseLoginDefs(file string) (*limits, error) {
	l := *defaultLimits
	if len(file) == 0 {
		return &l, nil
	}
	var mapping = map[string]*int64{
		"SYS_UID_MIN": &l.minUID,
		"SYS_UID_MAX": &l.maxUID,
		"SYS_GID_MIN": &l.minGID,
		"SYS_GID_MAX": &l.maxGID,
	}
	lines := strings.Split(file, "\n")
	for i, line := range lines {
		for k, v := range mapping {
			// A line must start with one of the definitions
			if !strings.HasPrefix(line, k) {
				continue
			}
			line = strings.TrimPrefix(line, k)
			line = strings.TrimSpace(line)
			val, err := strconv.ParseInt(line, 10, 64)
			if err != nil {
				return nil, errors.Wrapf(err, "could not parse value for %s at line %d", k, i)
			}
			*v = val
		}
	}
	return &l, nil
}


// parseEntries can be used to parse an /etc/passwd or /etc/group file as their format is similar.
// It returns a slice of entries obtained from the file.
// https://www.cyberciti.biz/faq/understanding-etcpasswd-file-format/
// https://www.cyberciti.biz/faq/understanding-etcgroup-file/
func parseEntries(file string, totalFields int) ([]*entry, error) {
	if totalFields != totalFieldsUser && totalFields != totalFieldsGroup {
		return nil, errors.Errorf("unsupported total fields for entry parsing: %d", totalFields)
	}
	lines := strings.Split(file, "\n")
	entries := []*entry{}
	for i, line := range lines {
		line = strings.TrimSpace(line)
		if len(line) == 0 {
			continue
		}
		fields := strings.Split(line, ":")
		if len(fields) != totalFields {
			return nil, errors.Errorf("entry must have %d fields separated by ':', "+
				"got %d at line %d: %s", totalFields, len(fields), i, line)
		}
		id, err := strconv.ParseInt(fields[2], 10, 64)
		if err != nil {
			return nil, errors.Wrapf(err, "error parsing id at line %d", i)
		}
		entry := &entry{name: fields[0], id: id}
		if totalFields == totalFieldsGroup {
			entry.userNames = strings.Split(fields[3], ",")
		} else {
			gid, err := strconv.ParseInt(fields[3], 10, 64)
			if err != nil {
				return nil, errors.Wrapf(err, "error parsing GID at line %d", i)
			}
			entry.gid = gid
			entry.shell = fields[6]
		}
		entries = append(entries, entry)
	}
	return entries, nil
}


// validateEntries takes user and group entries and validates if these entries are valid based on limits,
// mapping between users and groups and specs. Returns slices of missing user and group entries that must be created.
// Returns an error if existing users and groups do not match requirements.
func validateEntries(users, groups []*entry, limits *limits) ([]*entry, []*entry, error) {
	u := []*entry{}
	g := []*entry{}
	// Validate users
	for _, uc := range usersToCreateSpec {
		for _, user := range users {
			if uc.name != user.name {
				continue
			}
			// Found existing user
			if user.id < limits.minUID || user.id > limits.maxUID {
				return nil, nil, errors.Errorf("UID %d for user %q is outside the system UID range: %d - %d",
					user.id, user.name, limits.minUID, limits.maxUID)
			}
			if user.shell != noshell {
				return nil, nil, errors.Errorf("user %q has unexpected shell %q; expected %q",
					user.name, user.shell, noshell)
			}
			for _, g := range groups {
				if g.id != user.gid {
					continue
				}
				// Found matching group GID for user GID
				if g.name != uc.name {
					return nil, nil, errors.Errorf("user %q has GID %d but the group with that GID is not named %q",
						uc.name, g.id, uc.name)
				}
				goto skipUser // Valid group GID and name; skip
			}
			return nil, nil, errors.Errorf("could not find group with GID %d for user %q", user.gid, user.name)
		}
		u = append(u, uc)
	skipUser:
	}
	// validate groups
	for _, gc := range groupsToCreateSpec {
		for _, group := range groups {
			if gc.name != group.name {
				continue
			}
			if group.id < limits.minGID || group.id > limits.maxGID {
				return nil, nil, errors.Errorf("GID %d for user %q is outside the system UID range: %d - %d",
					group.id, group.name, limits.minGID, limits.maxGID)
			}
			u1 := strings.Join(gc.userNames, ",")
			u2 := strings.Join(group.userNames, ",")
			if u1 != u2 {
				return nil, nil, errors.Errorf("expected users %q for group %q; got %q",
					u1, gc.name, u2)
			}
			goto skipGroup // group has valid users; skip
		}
		g = append(g, gc)
	skipGroup:
	}
	return u, g, nil
}


// allocateIDs takes a list of entries and based on minimum and maximum ID allocates a "total" of IDs.
func allocateIDs(entries []*entry, min, max int64, total int) ([]int64, error) {
	if total == 0 {
		return []int64{}, nil
	}
	ids := make([]int64, 0, total)
	for i := min; i < max+1; i++ {
		i64 := int64(i)
		for _, e := range entries {
			if i64 == e.id {
				goto continueLoop
			}
		}
		ids = append(ids, i64)
		if len(ids) == total {
			return ids, nil
		}
	continueLoop:
	}
	return nil, errors.Errorf("could not allocate %d IDs based on existing entries in the range: %d - %d",
		total, min, max)
}


// addEntries takes /etc/passwd or /etc/group file content and appends entries to it based
// on a createEntry function. Returns the updated contents of the file.
func addEntries(file string, entries []*entry, createEntry func(*entry) string) string {
	out := file
	newLines := make([]string, 0, len(entries))
	for _, e := range entries {
		newLines = append(newLines, createEntry(e))
	}
	newLinesStr := ""
	if len(newLines) > 0 {
		if !strings.HasSuffix(out, "\n") { // Append a new line if its missing.
			newLinesStr = "\n"
		}
		newLinesStr += strings.Join(newLines, "\n") + "\n"
	}
	return out + newLinesStr
}


// removeEntries takes /etc/passwd or /etc/group file content and deletes entries from them
// by name matching. Returns the updated contents of the file and the number of entries removed.
func removeEntries(file string, entries []*entry) (string, int) {
	lines := strings.Split(file, "\n")
	total := len(lines) - len(entries)
	if total < 0 {
		total = 0
	}
	newLines := make([]string, 0, total)
	removed := 0
	for _, line := range lines {
		for _, entry := range entries {
			if strings.HasPrefix(line, entry.name+":") {
				removed++
				goto continueLoop
			}
		}
		newLines = append(newLines, line)
	continueLoop:
	}
	return strings.Join(newLines, "\n"), removed
}


// assignUserAndGroupIDs takes the list of existing groups, the users and groups to be created,
// and assigns UIDs and GIDs to the users and groups to be created based on a list of provided UIDs and GIDs.
// Returns an error if not enough UIDs or GIDs are passed. It does not perform any other validation.
func assignUserAndGroupIDs(groups, usersToCreate, groupsToCreate []*entry, uids, gids []int64) error {
	if len(gids) < len(groupsToCreate) {
		return errors.Errorf("not enough GIDs to assign to groups: have %d, want %d", len(gids), len(groupsToCreate))
	}
	if len(uids) < len(usersToCreate) {
		return errors.Errorf("not enough UIDs to assign to users: have %d, want %d", len(uids), len(usersToCreate))
	}
	for i := range groupsToCreate {
		groupsToCreate[i].id = gids[i]
	}
	// Concat the list of old and new groups to find a matching GID.
	groupsConcat := append([]*entry{}, groups...)
	groupsConcat = append(groupsConcat, groupsToCreate...)
	for i := range usersToCreate {
		usersToCreate[i].id = uids[i]
		for _, g := range groupsConcat {
			if usersToCreate[i].name == g.name {
				usersToCreate[i].gid = g.id
				break
			}
		}
	}
	return nil
}


func createGroup(e *entry) string

// createGroup is a helper function to produce a group from entry.
func createGroup(e *entry) string {
	return fmt.Sprintf("%s:x:%d:%s", e.name, e.id, strings.Join(e.userNames, ","))
}


func createUser(e *entry) string

// createUser is a helper function to produce a user from entry.
func createUser(e *entry) string {
	return fmt.Sprintf("%s:x:%d:%d:::/bin/false", e.name, e.id, e.gid)
}


// entriesToEntryMap takes a list of entries and prepares an EntryMap object.
func entriesToEntryMap(entries, spec []*entry) (*EntryMap, error) {
	m := map[string]*entry{}
	for _, spec := range spec {
		for _, e := range entries {
			if spec.name == e.name {
				entry := *e
				m[e.name] = &entry
				goto continueLoop
			}
		}
		return nil, errors.Errorf("could not find entry %q in the list", spec.name)
	continueLoop:
	}
	return &EntryMap{entries: m}, nil
}


// entriesToString is a utility to convert a list of entries to string.
func entriesToString(entries []*entry) string {
	lines := make([]string, 0, len(entries))
	for _, e := range entries {
		lines = append(lines, e.name)
	}
	sort.Strings(lines)
	return strings.Join(lines, ",")
}


// openFileWithLock opens the file at path by acquiring an exclive write lock.
// The returned close() function should be called to release the lock and close the file.
// If a lock cannot be obtained the function fails after a period of time.
func openFileWithLock(path string) (f *os.File, close func(), err error) {
	f, err = os.OpenFile(path, os.O_RDWR, os.ModePerm)
	if err != nil {
		return nil, nil, err
	}
	deadline := time.Now().Add(time.Second * 5)
	for {
		// If another process is holding a write lock, this call will exit
		// with an error. F_SETLK is used instead of F_SETLKW to avoid
		// the case where a runaway process grabs the exclusive lock and
		// blocks this call indefinitely.
		// https://man7.org/linux/man-pages/man2/fcntl.2.html
		lock := syscall.Flock_t{Type: syscall.F_WRLCK}
		if err = syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &lock); err == nil {
			break
		}
		time.Sleep(200 * time.Millisecond)
		if time.Now().After(deadline) {
			err = errors.Wrapf(err, "timeout attempting to obtain lock on file %q", path)
			break
		}
	}
	if err != nil {
		f.Close()
		return nil, nil, err
	}
	close = func() {
		// This function should be called once operations with the file are finished.
		// It unlocks the file and closes it.
		unlock := syscall.Flock_t{Type: syscall.F_UNLCK}
		syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &unlock)
		f.Close()
	}
	return f, close, nil
}


// readFile reads a File into a string.
func readFile(f *os.File) (string, error) {
	buf := bytes.NewBuffer(nil)
	if _, err := f.Seek(0, io.SeekStart); err != nil {
		return "", err
	}
	if _, err := io.Copy(buf, f); err != nil {
		return "", err
	}
	return buf.String(), nil
}


// writeFile writes a string to a File.
func writeFile(f *os.File, str string) error {
	if _, err := f.Seek(0, io.SeekStart); err != nil {
		return err
	}
	if _, err := f.Write([]byte(str)); err != nil {
		return err
	}
	if err := f.Truncate(int64(len(str))); err != nil {
		return err
	}
	return nil
}


// UpdatePathOwnerAndPermissions updates the owner and permissions of the given path.
// If the path is a directory it is not recursively updated.
func UpdatePathOwnerAndPermissions(path string, uid, gid int64, perms uint32) error {
	if err := os.Chown(path, int(uid), int(gid)); err != nil {
		return errors.Wrapf(err, "failed to update owner of %q to uid: %d and gid: %d", path, uid, gid)
	}
	fm := os.FileMode(perms)
	if err := os.Chmod(path, fm); err != nil {
		return errors.Wrapf(err, "failed to update permissions of %q to %s", path, fm.String())
	}
	return nil
}


// UpdatePathOwner recursively updates the owners of a directory.
// It is equivalent to calling `chown -R uid:gid /path/to/dir`.
func UpdatePathOwner(dirPath string, uid, gid int64) error {
	err := filepath.WalkDir(dirPath, func(path string, d os.DirEntry, err error) error {
		if err := os.Chown(path, int(uid), int(gid)); err != nil {
			return errors.Wrapf(err, "failed to update owner of %q to uid: %d and gid: %d", path, uid, gid)
		}
		return nil
	})
	return err
}


func (*EntryMap) ID(string) *int64

// ID is a NO-OP on non-Linux.
func (*EntryMap) ID(string) *int64 {
	return nil
}


func (*EntryMap) String() string

// String is NO-OP on non-Linux.
func (*EntryMap) String() string {
	return ""
}


// AddUsersAndGroups is a NO-OP on non-Linux.
func AddUsersAndGroups() (*UsersAndGroups, error) {
	return nil, nil
}


func RemoveUsersAndGroups() error

// RemoveUsersAndGroups is a NO-OP on non-Linux.
func RemoveUsersAndGroups() error {
	return nil
}


// UpdatePathOwnerAndPermissions is a NO-OP on non-Linux.
func UpdatePathOwnerAndPermissions(path string, uid, gid int64, perms uint32) error {
	return nil
}


// UpdatePathOwner is a NO-OP on non-Linux.
func UpdatePathOwner(dirPath string, uid, gid int64) error {
	return nil
}


// KubernetesReleaseVersion is helper function that can fetch
// available version information from release servers based on
// label names, like "stable" or "latest".
//
// If argument is already semantic version string, it
// will return same string.
//
// In case of labels, it tries to fetch from release
// servers and then return actual semantic version.
//
// Available names on release servers:
//
//	stable      (latest stable release)
//	stable-1    (latest stable release in 1.x)
//	stable-1.0  (and similarly 1.1, 1.2, 1.3, ...)
//	latest      (latest release, including alpha/beta)
//	latest-1    (latest release in 1.x, including alpha/beta)
//	latest-1.0  (and similarly 1.1, 1.2, 1.3, ...)
func KubernetesReleaseVersion(version string) (string, error) {
	return kubernetesReleaseVersion(version, fetchFromURL)
}


// kubernetesReleaseVersion is a helper function to fetch
// available version information. Used for testing to eliminate
// the need for internet calls.
func kubernetesReleaseVersion(version string, fetcher func(string, time.Duration) (string, error)) (string, error) {
	ver := normalizedBuildVersion(version)
	if len(ver) != 0 {
		return ver, nil
	}

	bucketURL, versionLabel, err := splitVersion(version)
	if err != nil {
		return "", err
	}

	// revalidate, if exact build from e.g. CI bucket requested.
	ver = normalizedBuildVersion(versionLabel)
	if len(ver) != 0 {
		return ver, nil
	}

	// kubeReleaseLabelRegex matches labels such as: latest, latest-1, latest-1.10
	if kubeReleaseLabelRegex.MatchString(versionLabel) {
		// Try to obtain a client version.
		// pkgversion.Get().String() should always return a correct version added by the golang
		// linker and the build system. The version can still be missing when doing unit tests
		// on individual packages.
		clientVersion, clientVersionErr := kubeadmVersion(pkgversion.Get().String())
		// Fetch version from the internet.
		url := fmt.Sprintf("%s/%s.txt", bucketURL, versionLabel)
		body, err := fetcher(url, getReleaseVersionTimeout)
		if err != nil {
			if clientVersionErr == nil {
				// Handle air-gapped environments by falling back to the client version.
				klog.Warningf("could not fetch a Kubernetes version from the internet: %v", err)
				klog.Warningf("falling back to the local client version: %s", clientVersion)
				return kubernetesReleaseVersion(clientVersion, fetcher)
			}
		}

		if clientVersionErr != nil {
			if err != nil {
				klog.Warningf("could not obtain neither client nor remote version; fall back to: %s", constants.CurrentKubernetesVersion)
				return kubernetesReleaseVersion(constants.CurrentKubernetesVersion.String(), fetcher)
			}

			klog.Warningf("could not obtain client version; using remote version: %s", body)
			return kubernetesReleaseVersion(body, fetcher)
		}

		// both the client and the remote version are obtained; validate them and pick a stable version
		body, err = validateStableVersion(body, clientVersion)
		if err != nil {
			return "", err
		}
		// Re-validate received version and return.
		return kubernetesReleaseVersion(body, fetcher)
	}
	return "", errors.Errorf("version %q doesn't match patterns for neither semantic version nor labels (stable, latest, ...)", version)
}


// KubernetesVersionToImageTag is helper function that replaces all
// non-allowed symbols in tag strings with underscores.
// Image tag can only contain lowercase and uppercase letters, digits,
// underscores, periods and dashes.
// Current usage is for CI images where all of symbols except '+' are valid,
// but function is for generic usage where input can't be always pre-validated.
func KubernetesVersionToImageTag(version string) string {
	allowed := regexp.MustCompile(`[^-\w.]`)
	return allowed.ReplaceAllString(version, "_")
}


// KubernetesIsCIVersion checks if user requested CI version
func KubernetesIsCIVersion(version string) bool {
	subs := kubeBucketPrefixes.FindAllStringSubmatch(version, 1)
	if len(subs) == 1 && len(subs[0]) == 4 && strings.HasPrefix(subs[0][2], "ci") {
		return true
	}
	return false
}


// Internal helper: returns normalized build version (with "v" prefix if needed)
// If input doesn't match known version pattern, returns empty string.
func normalizedBuildVersion(version string) string {
	if kubeReleaseRegex.MatchString(version) {
		if strings.HasPrefix(version, "v") {
			return version
		}
		return "v" + version
	}
	return ""
}


// Internal helper: split version parts,
// Return base URL and cleaned-up version
func splitVersion(version string) (string, string, error) {
	var bucketURL, urlSuffix string
	subs := kubeBucketPrefixes.FindAllStringSubmatch(version, 1)
	if len(subs) != 1 || len(subs[0]) != 4 {
		return "", "", errors.Errorf("invalid version %q", version)
	}

	switch {
	case strings.HasPrefix(subs[0][2], "ci"):
		// Just use whichever the user specified
		urlSuffix = subs[0][2]
		bucketURL = kubeCIBucketURL
	default:
		urlSuffix = "release"
		bucketURL = kubeReleaseBucketURL
	}
	url := fmt.Sprintf("%s/%s", bucketURL, urlSuffix)
	return url, subs[0][3], nil
}


// Internal helper: return content of URL
func fetchFromURL(url string, timeout time.Duration) (string, error) {
	klog.V(2).Infof("fetching Kubernetes version from URL: %s", url)
	client := &http.Client{Timeout: timeout, Transport: netutil.SetOldTransportDefaults(&http.Transport{})}
	resp, err := client.Get(url)
	if err != nil {
		return "", errors.Errorf("unable to get URL %q: %s", url, err.Error())
	}
	defer resp.Body.Close()
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return "", errors.Errorf("unable to read content of URL %q: %s", url, err.Error())
	}
	bodyString := strings.TrimSpace(string(body))

	if resp.StatusCode != http.StatusOK {
		msg := fmt.Sprintf("unable to fetch file. URL: %q, status: %v", url, resp.Status)
		return bodyString, errors.New(msg)
	}
	return bodyString, nil
}


// kubeadmVersion returns the version of the client without metadata.
func kubeadmVersion(info string) (string, error) {
	v, err := versionutil.ParseSemantic(info)
	if err != nil {
		return "", errors.Wrap(err, "kubeadm version error")
	}
	// There is no utility in versionutil to get the version without the metadata,
	// so this needs some manual formatting.
	// Discard offsets after a release label and keep the labels down to e.g. `alpha.0` instead of
	// including the offset e.g. `alpha.0.206`. This is done to comply with GCR image tags.
	pre := v.PreRelease()
	patch := v.Patch()
	if len(pre) > 0 {
		if patch > 0 {
			// If the patch version is more than zero, decrement it and remove the label.
			// this is done to comply with the latest stable patch release.
			patch = patch - 1
			pre = ""
		} else {
			split := strings.Split(pre, ".")
			if len(split) > 2 {
				pre = split[0] + "." + split[1] // Exclude the third element
			} else if len(split) < 2 {
				pre = split[0] + ".0" // Append .0 to a partial label
			}
			pre = "-" + pre
		}
	}
	vStr := fmt.Sprintf("v%d.%d.%d%s", v.Major(), v.Minor(), patch, pre)
	return vStr, nil
}


// Validate if the remote version is one Minor release newer than the client version.
// This is done to conform with "stable-X" and only allow remote versions from
// the same Patch level release.
func validateStableVersion(remoteVersion, clientVersion string) (string, error) {
	verRemote, err := versionutil.ParseGeneric(remoteVersion)
	if err != nil {
		return "", errors.Wrap(err, "remote version error")
	}
	verClient, err := versionutil.ParseGeneric(clientVersion)
	if err != nil {
		return "", errors.Wrap(err, "client version error")
	}
	// If the remote Major version is bigger or if the Major versions are the same,
	// but the remote Minor is bigger use the client version release. This handles Major bumps too.
	if verClient.Major() < verRemote.Major() ||
		(verClient.Major() == verRemote.Major()) && verClient.Minor() < verRemote.Minor() {
		estimatedRelease := fmt.Sprintf("stable-%d.%d", verClient.Major(), verClient.Minor())
		klog.Infof("remote version is much newer: %s; falling back to: %s", remoteVersion, estimatedRelease)
		return estimatedRelease, nil
	}
	return remoteVersion, nil
}


func main()

func main() {
	kubeadmutil.CheckErr(app.Run())
}


func main() {
	flags := pflag.NewFlagSet("kubectl-convert", pflag.ExitOnError)
	pflag.CommandLine = flags

	kubeConfigFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag()
	kubeConfigFlags.AddFlags(flags)
	matchVersionKubeConfigFlags := cmdutil.NewMatchVersionFlags(kubeConfigFlags)

	f := cmdutil.NewFactory(matchVersionKubeConfigFlags)
	cmd := convert.NewCmdConvert(f, genericclioptions.IOStreams{In: os.Stdin, Out: os.Stdout, ErrOut: os.Stderr})
	matchVersionKubeConfigFlags.AddFlags(cmd.PersistentFlags())
	code := cli.Run(cmd)
	os.Exit(code)
}


func main() {
	command := cmd.NewDefaultKubectlCommand()
	if err := cli.RunNoErrOutput(command); err != nil {
		// Pretty-print the error and exit with an error.
		util.CheckErr(err)
	}
}


// BuildAuth creates an authenticator, an authorizer, and a matching authorizer attributes getter compatible with the kubelet's needs
// It returns AuthInterface, a run method to start internal controllers (like cert reloading) and error.
func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubeletconfig.KubeletConfiguration) (server.AuthInterface, func(<-chan struct{}), error) {
	// Get clients, if provided
	var (
		tokenClient authenticationclient.AuthenticationV1Interface
		sarClient   authorizationclient.AuthorizationV1Interface
	)
	if client != nil && !reflect.ValueOf(client).IsNil() {
		tokenClient = client.AuthenticationV1()
		sarClient = client.AuthorizationV1()
	}

	authenticator, runAuthenticatorCAReload, err := BuildAuthn(tokenClient, config.Authentication)
	if err != nil {
		return nil, nil, err
	}

	attributes := server.NewNodeAuthorizerAttributesGetter(nodeName)

	authorizer, err := BuildAuthz(sarClient, config.Authorization)
	if err != nil {
		return nil, nil, err
	}

	return server.NewKubeletAuth(authenticator, attributes, authorizer), runAuthenticatorCAReload, nil
}


// BuildAuthn creates an authenticator compatible with the kubelet's needs
func BuildAuthn(client authenticationclient.AuthenticationV1Interface, authn kubeletconfig.KubeletAuthentication) (authenticator.Request, func(<-chan struct{}), error) {
	var dynamicCAContentFromFile *dynamiccertificates.DynamicFileCAContent
	var err error
	if len(authn.X509.ClientCAFile) > 0 {
		dynamicCAContentFromFile, err = dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", authn.X509.ClientCAFile)
		if err != nil {
			return nil, nil, err
		}
	}

	authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{
		Anonymous:                          authn.Anonymous.Enabled,
		CacheTTL:                           authn.Webhook.CacheTTL.Duration,
		ClientCertificateCAContentProvider: dynamicCAContentFromFile,
	}

	if authn.Webhook.Enabled {
		if client == nil {
			return nil, nil, errors.New("no client provided, cannot use webhook authentication")
		}
		authenticatorConfig.WebhookRetryBackoff = genericoptions.DefaultAuthWebhookRetryBackoff()
		authenticatorConfig.TokenAccessReviewClient = client
	}

	authenticator, _, err := authenticatorConfig.New()
	if err != nil {
		return nil, nil, err
	}

	return authenticator, func(stopCh <-chan struct{}) {
		// generate a context from stopCh. This is to avoid modifying files which are relying on this method
		// TODO: See if we can pass ctx to the current method
		ctx, cancel := context.WithCancel(context.Background())
		go func() {
			select {
			case <-stopCh:
				cancel() // stopCh closed, so cancel our context
			case <-ctx.Done():
			}
		}()
		if dynamicCAContentFromFile != nil {
			go dynamicCAContentFromFile.Run(ctx, 1)
		}
	}, err
}


// BuildAuthz creates an authorizer compatible with the kubelet's needs
func BuildAuthz(client authorizationclient.AuthorizationV1Interface, authz kubeletconfig.KubeletAuthorization) (authorizer.Authorizer, error) {
	switch authz.Mode {
	case kubeletconfig.KubeletAuthorizationModeAlwaysAllow:
		return authorizerfactory.NewAlwaysAllowAuthorizer(), nil

	case kubeletconfig.KubeletAuthorizationModeWebhook:
		if client == nil {
			return nil, errors.New("no client provided, cannot use webhook authorization")
		}
		authorizerConfig := authorizerfactory.DelegatingAuthorizerConfig{
			SubjectAccessReviewClient: client,
			AllowCacheTTL:             authz.Webhook.CacheAuthorizedTTL.Duration,
			DenyCacheTTL:              authz.Webhook.CacheUnauthorizedTTL.Duration,
			WebhookRetryBackoff:       genericoptions.DefaultAuthWebhookRetryBackoff(),
		}
		return authorizerConfig.New()

	case "":
		return nil, fmt.Errorf("no authorization mode specified")

	default:
		return nil, fmt.Errorf("unknown authorization mode %s", authz.Mode)

	}
}


func initForOS(service bool, priorityClass string) error {
	return nil
}


// getPriorityValue returns the value associated with a Windows process priorityClass
// Ref: https://docs.microsoft.com/en-us/windows/win32/cimwin32prov/setpriority-method-in-class-win32-process
func getPriorityValue(priorityClassName string) uint32 {
	var priorityClassMap = map[string]uint32{
		"IDLE_PRIORITY_CLASS":         uint32(windows.IDLE_PRIORITY_CLASS),
		"BELOW_NORMAL_PRIORITY_CLASS": uint32(windows.BELOW_NORMAL_PRIORITY_CLASS),
		"NORMAL_PRIORITY_CLASS":       uint32(windows.NORMAL_PRIORITY_CLASS),
		"ABOVE_NORMAL_PRIORITY_CLASS": uint32(windows.ABOVE_NORMAL_PRIORITY_CLASS),
		"HIGH_PRIORITY_CLASS":         uint32(windows.HIGH_PRIORITY_CLASS),
		"REALTIME_PRIORITY_CLASS":     uint32(windows.REALTIME_PRIORITY_CLASS),
	}
	return priorityClassMap[priorityClassName]
}


// createWindowsJobObject creates a new Job Object
// (https://docs.microsoft.com/en-us/windows/win32/procthread/job-objects),
// and specifies the priority class for the job object to the specified value.
// A job object is used here so that any spawned processes such as powershell or
// wmic are created at the specified thread priority class.
// Running kubelet with above normal / high priority  can help improve
// responsiveness on machines with high CPU utilization.
func createWindowsJobObject(pc uint32) (windows.Handle, error) {
	job, err := windows.CreateJobObject(nil, nil)
	if err != nil {
		return windows.InvalidHandle, fmt.Errorf("windows.CreateJobObject failed: %w", err)
	}
	limitInfo := windows.JOBOBJECT_BASIC_LIMIT_INFORMATION{
		LimitFlags:    windows.JOB_OBJECT_LIMIT_PRIORITY_CLASS,
		PriorityClass: pc,
	}
	if _, err := windows.SetInformationJobObject(
		job,
		windows.JobObjectBasicLimitInformation,
		uintptr(unsafe.Pointer(&limitInfo)),
		uint32(unsafe.Sizeof(limitInfo))); err != nil {
		return windows.InvalidHandle, fmt.Errorf("windows.SetInformationJobObject failed: %w", err)
	}
	return job, nil
}


func initForOS(windowsService bool, windowsPriorityClass string) error {
	priority := getPriorityValue(windowsPriorityClass)
	if priority == 0 {
		return fmt.Errorf("unknown priority class %s, valid ones are available at "+
			"https://docs.microsoft.com/en-us/windows/win32/procthread/scheduling-priorities", windowsPriorityClass)
	}
	klog.InfoS("Creating a Windows job object and adding kubelet process to it", "windowsPriorityClass", windowsPriorityClass)
	job, err := createWindowsJobObject(priority)
	if err != nil {
		return err
	}
	if err := windows.AssignProcessToJobObject(job, windows.CurrentProcess()); err != nil {
		return fmt.Errorf("windows.AssignProcessToJobObject failed: %w", err)
	}

	if windowsService {
		return service.InitService(serviceName)
	}
	return nil
}


// NewContainerRuntimeOptions will create a new ContainerRuntimeOptions with
// default values.
func NewContainerRuntimeOptions() *config.ContainerRuntimeOptions {
	return &config.ContainerRuntimeOptions{
		PodSandboxImage: defaultPodSandboxImage,
	}
}


func AddGlobalFlags(fs *pflag.FlagSet)

// AddGlobalFlags explicitly registers flags that libraries (glog, verflag, etc.) register
// against the global flagsets from "flag" and "github.com/spf13/pflag".
// We do this in order to prevent unwanted flags from leaking into the Kubelet's flagset.
func AddGlobalFlags(fs *pflag.FlagSet) {
	addCadvisorFlags(fs)
	addCredentialProviderFlags(fs)
	verflag.AddFlags(fs)
	logs.AddFlags(fs, logs.SkipLoggingConfigurationFlags())
}


func normalize(s string) string

// normalize replaces underscores with hyphens
// we should always use hyphens instead of underscores when registering kubelet flags
func normalize(s string) string {
	return strings.Replace(s, "_", "-", -1)
}


// register adds a flag to local that targets the Value associated with the Flag named globalName in global
func register(global *flag.FlagSet, local *pflag.FlagSet, globalName string) {
	if f := global.Lookup(globalName); f != nil {
		pflagFlag := pflag.PFlagFromGoFlag(f)
		pflagFlag.Name = normalize(pflagFlag.Name)
		local.AddFlag(pflagFlag)
	} else {
		panic(fmt.Sprintf("failed to find flag in global flagset (flag): %s", globalName))
	}
}


// pflagRegister adds a flag to local that targets the Value associated with the Flag named globalName in global
func pflagRegister(global, local *pflag.FlagSet, globalName string) {
	if f := global.Lookup(globalName); f != nil {
		f.Name = normalize(f.Name)
		local.AddFlag(f)
	} else {
		panic(fmt.Sprintf("failed to find flag in global flagset (pflag): %s", globalName))
	}
}


// registerDeprecated registers the flag with register, and then marks it deprecated
func registerDeprecated(global *flag.FlagSet, local *pflag.FlagSet, globalName, deprecated string) {
	register(global, local, globalName)
	local.Lookup(normalize(globalName)).Deprecated = deprecated
}


// addCredentialProviderFlags adds flags from k8s.io/kubernetes/pkg/credentialprovider
func addCredentialProviderFlags(fs *pflag.FlagSet) {
	// lookup flags in global flag set and re-register the values with our flagset
	global := pflag.CommandLine
	local := pflag.NewFlagSet(os.Args[0], pflag.ExitOnError)

	addLegacyCloudProviderCredentialProviderFlags(global, local)

	fs.AddFlagSet(local)
}


// addCadvisorFlags adds flags from cadvisor
func addCadvisorFlags(fs *pflag.FlagSet) {
	// lookup flags in global flag set and re-register the values with our flagset
	global := flag.CommandLine
	local := pflag.NewFlagSet(os.Args[0], pflag.ExitOnError)

	// e2e node tests rely on this
	register(global, local, "housekeeping_interval")

	// These flags were implicit from cadvisor, and are mistakes that should be registered deprecated:
	const deprecated = "This is a cadvisor flag that was mistakenly registered with the Kubelet. Due to legacy concerns, it will follow the standard CLI deprecation timeline before being removed."

	registerDeprecated(global, local, "application_metrics_count_limit", deprecated)
	registerDeprecated(global, local, "boot_id_file", deprecated)
	registerDeprecated(global, local, "container_hints", deprecated)
	registerDeprecated(global, local, "containerd", deprecated)
	registerDeprecated(global, local, "enable_load_reader", deprecated)
	registerDeprecated(global, local, "event_storage_age_limit", deprecated)
	registerDeprecated(global, local, "event_storage_event_limit", deprecated)
	registerDeprecated(global, local, "global_housekeeping_interval", deprecated)
	registerDeprecated(global, local, "log_cadvisor_usage", deprecated)
	registerDeprecated(global, local, "machine_id_file", deprecated)
	registerDeprecated(global, local, "storage_driver_user", deprecated)
	registerDeprecated(global, local, "storage_driver_password", deprecated)
	registerDeprecated(global, local, "storage_driver_host", deprecated)
	registerDeprecated(global, local, "storage_driver_db", deprecated)
	registerDeprecated(global, local, "storage_driver_table", deprecated)
	registerDeprecated(global, local, "storage_driver_secure", deprecated)
	registerDeprecated(global, local, "storage_driver_buffer_duration", deprecated)
	registerDeprecated(global, local, "containerd-namespace", deprecated)

	// finally, add cadvisor flags to the provided flagset
	fs.AddFlagSet(local)
}


func addCadvisorFlags(fs *pflag.FlagSet) {
}


func addLegacyCloudProviderCredentialProviderFlags(global, local *pflag.FlagSet) {
	// no-op when no legacy providers are compiled in
}


func addLegacyCloudProviderCredentialProviderFlags(global, local *pflag.FlagSet) {
	// TODO(#58034): This is not a static file, so it's not quite as straightforward as --google-json-key.
	// We need to figure out how ACR users can dynamically provide pull credentials before we can deprecate this.
	pflagRegister(global, local, "azure-container-registry-config")
}


func NewKubeletFlags() *KubeletFlags

// NewKubeletFlags will create a new KubeletFlags with default values
func NewKubeletFlags() *KubeletFlags {
	return &KubeletFlags{
		ContainerRuntimeOptions: *NewContainerRuntimeOptions(),
		CertDirectory:           "/var/lib/kubelet/pki",
		RootDirectory:           filepath.Clean(defaultRootDir),
		MaxContainerCount:       -1,
		MaxPerPodContainerCount: 1,
		MinimumGCAge:            metav1.Duration{Duration: 0},
		RegisterSchedulable:     true,
		NodeLabels:              make(map[string]string),
	}
}


// ValidateKubeletFlags validates Kubelet's configuration flags and returns an error if they are invalid.
func ValidateKubeletFlags(f *KubeletFlags) error {
	unknownLabels := sets.NewString()
	invalidLabelErrs := make(map[string][]string)
	for k, v := range f.NodeLabels {
		if isKubernetesLabel(k) && !kubeletapis.IsKubeletLabel(k) {
			unknownLabels.Insert(k)
		}

		if errs := validation.IsQualifiedName(k); len(errs) > 0 {
			invalidLabelErrs[k] = append(invalidLabelErrs[k], errs...)
		}
		if errs := validation.IsValidLabelValue(v); len(errs) > 0 {
			invalidLabelErrs[v] = append(invalidLabelErrs[v], errs...)
		}
	}
	if len(unknownLabels) > 0 {
		return fmt.Errorf("unknown 'kubernetes.io' or 'k8s.io' labels specified with --node-labels: %v\n--node-labels in the 'kubernetes.io' namespace must begin with an allowed prefix (%s) or be in the specifically allowed set (%s)", unknownLabels.List(), strings.Join(kubeletapis.KubeletLabelNamespaces(), ", "), strings.Join(kubeletapis.KubeletLabels(), ", "))
	}
	if len(invalidLabelErrs) > 0 {
		labelErrs := []string{}
		for k, v := range invalidLabelErrs {
			labelErrs = append(labelErrs, fmt.Sprintf("'%s' - %s", k, strings.Join(v, ", ")))
		}
		return fmt.Errorf("invalid node labels: %s", strings.Join(labelErrs, "; "))
	}
	return nil
}


func isKubernetesLabel(key string) bool

func isKubernetesLabel(key string) bool {
	namespace := getLabelNamespace(key)
	if namespace == "kubernetes.io" || strings.HasSuffix(namespace, ".kubernetes.io") {
		return true
	}
	if namespace == "k8s.io" || strings.HasSuffix(namespace, ".k8s.io") {
		return true
	}
	return false
}


func getLabelNamespace(key string) string {
	if parts := strings.SplitN(key, "/", 2); len(parts) == 2 {
		return parts[0]
	}
	return ""
}


// NewKubeletConfiguration will create a new KubeletConfiguration with default values
func NewKubeletConfiguration() (*kubeletconfig.KubeletConfiguration, error) {
	scheme, _, err := kubeletscheme.NewSchemeAndCodecs()
	if err != nil {
		return nil, err
	}
	versioned := &v1beta1.KubeletConfiguration{}
	scheme.Default(versioned)
	config := &kubeletconfig.KubeletConfiguration{}
	if err := scheme.Convert(versioned, config, nil); err != nil {
		return nil, err
	}
	applyLegacyDefaults(config)
	return config, nil
}


// applyLegacyDefaults applies legacy default values to the KubeletConfiguration in order to
// preserve the command line API. This is used to construct the baseline default KubeletConfiguration
// before the first round of flag parsing.
func applyLegacyDefaults(kc *kubeletconfig.KubeletConfiguration) {
	// --anonymous-auth
	kc.Authentication.Anonymous.Enabled = true
	// --authentication-token-webhook
	kc.Authentication.Webhook.Enabled = false
	// --authorization-mode
	kc.Authorization.Mode = kubeletconfig.KubeletAuthorizationModeAlwaysAllow
	// --read-only-port
	kc.ReadOnlyPort = ports.KubeletReadOnlyPort
}


// NewKubeletServer will create a new KubeletServer with default values.
func NewKubeletServer() (*KubeletServer, error) {
	config, err := NewKubeletConfiguration()
	if err != nil {
		return nil, err
	}
	return &KubeletServer{
		KubeletFlags:         *NewKubeletFlags(),
		KubeletConfiguration: *config,
	}, nil
}


// ValidateKubeletServer validates configuration of KubeletServer and returns an error if the input configuration is invalid.
func ValidateKubeletServer(s *KubeletServer) error {
	// please add any KubeletConfiguration validation to the kubeletconfigvalidation.ValidateKubeletConfiguration function
	if err := kubeletconfigvalidation.ValidateKubeletConfiguration(&s.KubeletConfiguration, utilfeature.DefaultFeatureGate); err != nil {
		return err
	}
	if err := ValidateKubeletFlags(&s.KubeletFlags); err != nil {
		return err
	}
	return nil
}


// AddFlags adds flags for a specific KubeletServer to the specified FlagSet
func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
	s.KubeletFlags.AddFlags(fs)
	AddKubeletConfigFlags(fs, &s.KubeletConfiguration)
}


// AddFlags adds flags for a specific KubeletFlags to the specified FlagSet
func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) {
	fs := pflag.NewFlagSet("", pflag.ExitOnError)
	defer func() {
		// Unhide deprecated flags. We want deprecated flags to show in Kubelet help.
		// We have some hidden flags, but we might as well unhide these when they are deprecated,
		// as silently deprecating and removing (even hidden) things is unkind to people who use them.
		fs.VisitAll(func(f *pflag.Flag) {
			if len(f.Deprecated) > 0 {
				f.Hidden = false
			}
		})
		mainfs.AddFlagSet(fs)
	}()

	f.ContainerRuntimeOptions.AddFlags(fs)
	f.addOSFlags(fs)

	fs.StringVar(&f.KubeletConfigFile, "config", f.KubeletConfigFile, "The Kubelet will load its initial configuration from this file. The path may be absolute or relative; relative paths start at the Kubelet's current working directory. Omit this flag to use the built-in default configuration values. Command-line flags override configuration from this file.")
	fs.StringVar(&f.KubeConfig, "kubeconfig", f.KubeConfig, "Path to a kubeconfig file, specifying how to connect to the API server. Providing --kubeconfig enables API server mode, omitting --kubeconfig enables standalone mode.")

	fs.StringVar(&f.BootstrapKubeconfig, "bootstrap-kubeconfig", f.BootstrapKubeconfig, "Path to a kubeconfig file that will be used to get client certificate for kubelet. "+
		"If the file specified by --kubeconfig does not exist, the bootstrap kubeconfig is used to request a client certificate from the API server. "+
		"On success, a kubeconfig file referencing the generated client certificate and key is written to the path specified by --kubeconfig. "+
		"The client certificate and key file will be stored in the directory pointed by --cert-dir.")

	fs.StringVar(&f.HostnameOverride, "hostname-override", f.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname. If --cloud-provider is set, the cloud provider determines the name of the node (consult cloud provider documentation to determine if and how the hostname is used).")

	fs.StringVar(&f.NodeIP, "node-ip", f.NodeIP, "IP address (or comma-separated dual-stack IP addresses) of the node. If unset, kubelet will use the node's default IPv4 address, if any, or its default IPv6 address if it has no IPv4 addresses. You can pass '::' to make it prefer the default IPv6 address rather than the default IPv4 address.")

	fs.StringVar(&f.CertDirectory, "cert-dir", f.CertDirectory, "The directory where the TLS certs are located. "+
		"If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.")

	fs.StringVar(&f.RootDirectory, "root-dir", f.RootDirectory, "Directory path for managing kubelet files (volume mounts,etc).")
	fs.BoolVar(&f.SeccompDefault, "seccomp-default", f.SeccompDefault, "Enable the use of `RuntimeDefault` as the default seccomp profile for all workloads.")

	// EXPERIMENTAL FLAGS
	bindableNodeLabels := cliflag.ConfigurationMap(f.NodeLabels)
	fs.Var(&bindableNodeLabels, "node-labels", fmt.Sprintf("<Warning: Alpha feature> Labels to add when registering the node in the cluster.  Labels must be key=value pairs separated by ','. Labels in the 'kubernetes.io' namespace must begin with an allowed prefix (%s) or be in the specifically allowed set (%s)", strings.Join(kubeletapis.KubeletLabelNamespaces(), ", "), strings.Join(kubeletapis.KubeletLabels(), ", ")))
	fs.StringVar(&f.LockFilePath, "lock-file", f.LockFilePath, "<Warning: Alpha feature> The path to file for kubelet to use as a lock file.")
	fs.BoolVar(&f.ExitOnLockContention, "exit-on-lock-contention", f.ExitOnLockContention, "Whether kubelet should exit upon lock-file contention.")

	// DEPRECATED FLAGS
	fs.DurationVar(&f.MinimumGCAge.Duration, "minimum-container-ttl-duration", f.MinimumGCAge.Duration, "Minimum age for a finished container before it is garbage collected.  Examples: '300ms', '10s' or '2h45m'")
	fs.MarkDeprecated("minimum-container-ttl-duration", "Use --eviction-hard or --eviction-soft instead. Will be removed in a future version.")
	fs.Int32Var(&f.MaxPerPodContainerCount, "maximum-dead-containers-per-container", f.MaxPerPodContainerCount, "Maximum number of old instances to retain per container.  Each container takes up some disk space.")
	fs.MarkDeprecated("maximum-dead-containers-per-container", "Use --eviction-hard or --eviction-soft instead. Will be removed in a future version.")
	fs.Int32Var(&f.MaxContainerCount, "maximum-dead-containers", f.MaxContainerCount, "Maximum number of old instances of containers to retain globally.  Each container takes up some disk space. To disable, set to a negative number.")
	fs.MarkDeprecated("maximum-dead-containers", "Use --eviction-hard or --eviction-soft instead. Will be removed in a future version.")
	fs.BoolVar(&f.RegisterSchedulable, "register-schedulable", f.RegisterSchedulable, "Register the node as schedulable. Won't have any effect if register-node is false.")
	fs.MarkDeprecated("register-schedulable", "will be removed in a future version")
	fs.BoolVar(&f.KeepTerminatedPodVolumes, "keep-terminated-pod-volumes", f.KeepTerminatedPodVolumes, "Keep terminated pod volumes mounted to the node after the pod terminates.  Can be useful for debugging volume related issues.")
	fs.MarkDeprecated("keep-terminated-pod-volumes", "will be removed in a future version")
	fs.StringVar(&f.ExperimentalMounterPath, "experimental-mounter-path", f.ExperimentalMounterPath, "[Experimental] Path of mounter binary. Leave empty to use the default mount.")
	fs.MarkDeprecated("experimental-mounter-path", "will be removed in 1.25 or later. in favor of using CSI.")
	fs.StringVar(&f.CloudProvider, "cloud-provider", f.CloudProvider, "The provider for cloud services. Set to empty string for running with no cloud provider. If set, the cloud provider determines the name of the node (consult cloud provider documentation to determine if and how the hostname is used).")
	fs.MarkDeprecated("cloud-provider", "will be removed in 1.25 or later, in favor of removing cloud provider code from Kubelet.")
	fs.StringVar(&f.CloudConfigFile, "cloud-config", f.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.")
	fs.MarkDeprecated("cloud-config", "will be removed in 1.25 or later, in favor of removing cloud provider code from Kubelet.")
	fs.BoolVar(&f.ExperimentalNodeAllocatableIgnoreEvictionThreshold, "experimental-allocatable-ignore-eviction", f.ExperimentalNodeAllocatableIgnoreEvictionThreshold, "When set to 'true', Hard Eviction Thresholds will be ignored while calculating Node Allocatable. See https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/ for more details. [default=false]")
	fs.MarkDeprecated("experimental-allocatable-ignore-eviction", "will be removed in 1.25 or later.")
}