getPatchSetsFromPath

// getPatchSetsFromPath walks a path, ignores sub-directories and non-patch files, and
// returns a list of patchFile objects.
func getPatchSetsFromPath(targetPath string, knownTargets []string, output io.Writer) ([]*patchSet, []string, []string, error) {
	patchFiles := []string{}
	ignoredFiles := []string{}
	patchSets := []*patchSet{}

	// Check if targetPath is a directory.
	info, err := os.Lstat(targetPath)
	if err != nil {
		goto return_path_error
	}
	if !info.IsDir() {
		err = &os.PathError{
			Op:   "getPatchSetsFromPath",
			Path: info.Name(),
			Err:  errors.New("not a directory"),
		}
		goto return_path_error
	}

	err = filepath.Walk(targetPath, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Sub-directories and "." are ignored.
		if info.IsDir() {
			return nil
		}

		baseName := info.Name()

		// Parse the filename and retrieve the target and patch type
		targetName, patchType, warn, err := parseFilename(baseName, knownTargets)
		if err != nil {
			return err
		}
		if warn != nil {
			fmt.Fprintf(output, "[patches] Ignoring file %q: %v\n", baseName, warn)
			ignoredFiles = append(ignoredFiles, baseName)
			return nil
		}

		// Read the patch file.
		data, err := os.ReadFile(path)
		if err != nil {
			return errors.Wrapf(err, "could not read the file %q", path)
		}

		if len(data) == 0 {
			fmt.Fprintf(output, "[patches] Ignoring empty file: %q\n", baseName)
			ignoredFiles = append(ignoredFiles, baseName)
			return nil
		}

		// Create a patchSet object.
		patchSet, err := createPatchSet(targetName, patchType, string(data))
		if err != nil {
			return err
		}

		patchFiles = append(patchFiles, baseName)
		patchSets = append(patchSets, patchSet)
		return nil
	})

return_path_error:
	if err != nil {
		return nil, nil, nil, errors.Wrapf(err, "could not list patch files for path %q", targetPath)
	}

	return patchSets, patchFiles, ignoredFiles, nil
}


NewCertificateAuthority

// NewCertificateAuthority creates new certificate and private key for the certificate authority
func NewCertificateAuthority(config *CertConfig) (*x509.Certificate, crypto.Signer, error) {
	key, err := NewPrivateKey(config.PublicKeyAlgorithm)
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to create private key while generating CA certificate")
	}

	cert, err := certutil.NewSelfSignedCACert(config.Config, key)
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to create self-signed CA certificate")
	}

	return cert, key, nil
}


NewIntermediateCertificateAuthority

// NewIntermediateCertificateAuthority creates new certificate and private key for an intermediate certificate authority
func NewIntermediateCertificateAuthority(parentCert *x509.Certificate, parentKey crypto.Signer, config *CertConfig) (*x509.Certificate, crypto.Signer, error) {
	key, err := NewPrivateKey(config.PublicKeyAlgorithm)
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to create private key while generating intermediate CA certificate")
	}

	cert, err := NewSignedCert(config, key, parentCert, parentKey, true)
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to sign intermediate CA certificate")
	}

	return cert, key, nil
}


NewCertAndKey

// NewCertAndKey creates new certificate and key by passing the certificate authority certificate and key
func NewCertAndKey(caCert *x509.Certificate, caKey crypto.Signer, config *CertConfig) (*x509.Certificate, crypto.Signer, error) {
	if len(config.Usages) == 0 {
		return nil, nil, errors.New("must specify at least one ExtKeyUsage")
	}

	key, err := NewPrivateKey(config.PublicKeyAlgorithm)
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to create private key")
	}

	cert, err := NewSignedCert(config, key, caCert, caKey, false)
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to sign certificate")
	}

	return cert, key, nil
}


NewCSRAndKey

// NewCSRAndKey generates a new key and CSR and that could be signed to create the given certificate
func NewCSRAndKey(config *CertConfig) (*x509.CertificateRequest, crypto.Signer, error) {
	key, err := NewPrivateKey(config.PublicKeyAlgorithm)
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to create private key")
	}

	csr, err := NewCSR(*config, key)
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to generate CSR")
	}

	return csr, key, nil
}


HasServerAuth

// HasServerAuth returns true if the given certificate is a ServerAuth
func HasServerAuth(cert *x509.Certificate) bool {
	for i := range cert.ExtKeyUsage {
		if cert.ExtKeyUsage[i] == x509.ExtKeyUsageServerAuth {
			return true
		}
	}
	return false
}


WriteCertAndKey

// WriteCertAndKey stores certificate and key at the specified location
func WriteCertAndKey(pkiPath string, name string, cert *x509.Certificate, key crypto.Signer) error {
	if err := WriteKey(pkiPath, name, key); err != nil {
		return errors.Wrap(err, "couldn't write key")
	}

	return WriteCert(pkiPath, name, cert)
}


WriteCert

// WriteCert stores the given certificate at the given location
func WriteCert(pkiPath, name string, cert *x509.Certificate) error {
	if cert == nil {
		return errors.New("certificate cannot be nil when writing to file")
	}

	certificatePath := pathForCert(pkiPath, name)
	if err := certutil.WriteCert(certificatePath, EncodeCertPEM(cert)); err != nil {
		return errors.Wrapf(err, "unable to write certificate to file %s", certificatePath)
	}

	return nil
}


WriteCertBundle

// WriteCertBundle stores the given certificate bundle at the given location
func WriteCertBundle(pkiPath, name string, certs []*x509.Certificate) error {
	for i, cert := range certs {
		if cert == nil {
			return errors.Errorf("found nil certificate at position %d when writing bundle to file", i)
		}
	}

	certificatePath := pathForCert(pkiPath, name)
	encoded, err := EncodeCertBundlePEM(certs)
	if err != nil {
		return errors.Wrapf(err, "unable to marshal certificate bundle to PEM")
	}
	if err := certutil.WriteCert(certificatePath, encoded); err != nil {
		return errors.Wrapf(err, "unable to write certificate bundle to file %s", certificatePath)
	}

	return nil
}


WriteKey

// WriteKey stores the given key at the given location
func WriteKey(pkiPath, name string, key crypto.Signer) error {
	if key == nil {
		return errors.New("private key cannot be nil when writing to file")
	}

	privateKeyPath := pathForKey(pkiPath, name)
	encoded, err := keyutil.MarshalPrivateKeyToPEM(key)
	if err != nil {
		return errors.Wrapf(err, "unable to marshal private key to PEM")
	}
	if err := keyutil.WriteKey(privateKeyPath, encoded); err != nil {
		return errors.Wrapf(err, "unable to write private key to file %s", privateKeyPath)
	}

	return nil
}


WriteCSR

// WriteCSR writes the pem-encoded CSR data to csrPath.
// The CSR file will be created with file mode 0600.
// If the CSR file already exists, it will be overwritten.
// The parent directory of the csrPath will be created as needed with file mode 0700.
func WriteCSR(csrDir, name string, csr *x509.CertificateRequest) error {
	if csr == nil {
		return errors.New("certificate request cannot be nil when writing to file")
	}

	csrPath := pathForCSR(csrDir, name)
	if err := os.MkdirAll(filepath.Dir(csrPath), os.FileMode(0700)); err != nil {
		return errors.Wrapf(err, "failed to make directory %s", filepath.Dir(csrPath))
	}

	if err := os.WriteFile(csrPath, EncodeCSRPEM(csr), os.FileMode(0600)); err != nil {
		return errors.Wrapf(err, "unable to write CSR to file %s", csrPath)
	}

	return nil
}


WritePublicKey

// WritePublicKey stores the given public key at the given location
func WritePublicKey(pkiPath, name string, key crypto.PublicKey) error {
	if key == nil {
		return errors.New("public key cannot be nil when writing to file")
	}

	publicKeyBytes, err := EncodePublicKeyPEM(key)
	if err != nil {
		return err
	}
	publicKeyPath := pathForPublicKey(pkiPath, name)
	if err := keyutil.WriteKey(publicKeyPath, publicKeyBytes); err != nil {
		return errors.Wrapf(err, "unable to write public key to file %s", publicKeyPath)
	}

	return nil
}


CertOrKeyExist

// CertOrKeyExist returns a boolean whether the cert or the key exists
func CertOrKeyExist(pkiPath, name string) bool {
	certificatePath, privateKeyPath := PathsForCertAndKey(pkiPath, name)

	_, certErr := os.Stat(certificatePath)
	_, keyErr := os.Stat(privateKeyPath)

	return !(os.IsNotExist(certErr) && os.IsNotExist(keyErr))
}


CSROrKeyExist

// CSROrKeyExist returns true if one of the CSR or key exists
func CSROrKeyExist(csrDir, name string) bool {
	csrPath := pathForCSR(csrDir, name)
	keyPath := pathForKey(csrDir, name)

	_, csrErr := os.Stat(csrPath)
	_, keyErr := os.Stat(keyPath)

	return !(os.IsNotExist(csrErr) && os.IsNotExist(keyErr))
}


TryLoadCertAndKeyFromDisk

// TryLoadCertAndKeyFromDisk tries to load a cert and a key from the disk and validates that they are valid
func TryLoadCertAndKeyFromDisk(pkiPath, name string) (*x509.Certificate, crypto.Signer, error) {
	cert, err := TryLoadCertFromDisk(pkiPath, name)
	if err != nil {
		return nil, nil, errors.Wrap(err, "failed to load certificate")
	}

	key, err := TryLoadKeyFromDisk(pkiPath, name)
	if err != nil {
		return nil, nil, errors.Wrap(err, "failed to load key")
	}

	return cert, key, nil
}


TryLoadCertFromDisk

// TryLoadCertFromDisk tries to load the cert from the disk
func TryLoadCertFromDisk(pkiPath, name string) (*x509.Certificate, error) {
	certificatePath := pathForCert(pkiPath, name)

	certs, err := certutil.CertsFromFile(certificatePath)
	if err != nil {
		return nil, errors.Wrapf(err, "couldn't load the certificate file %s", certificatePath)
	}

	// Safely pick the first one because the sender's certificate must come first in the list.
	// For details, see: https://www.rfc-editor.org/rfc/rfc4346#section-7.4.2
	cert := certs[0]

	return cert, nil
}


TryLoadCertChainFromDisk

// TryLoadCertChainFromDisk tries to load the cert chain from the disk
func TryLoadCertChainFromDisk(pkiPath, name string) (*x509.Certificate, []*x509.Certificate, error) {
	certificatePath := pathForCert(pkiPath, name)

	certs, err := certutil.CertsFromFile(certificatePath)
	if err != nil {
		return nil, nil, errors.Wrapf(err, "couldn't load the certificate file %s", certificatePath)
	}

	cert := certs[0]
	intermediates := certs[1:]

	return cert, intermediates, nil
}


TryLoadKeyFromDisk

// TryLoadKeyFromDisk tries to load the key from the disk and validates that it is valid
func TryLoadKeyFromDisk(pkiPath, name string) (crypto.Signer, error) {
	privateKeyPath := pathForKey(pkiPath, name)

	// Parse the private key from a file
	privKey, err := keyutil.PrivateKeyFromFile(privateKeyPath)
	if err != nil {
		return nil, errors.Wrapf(err, "couldn't load the private key file %s", privateKeyPath)
	}

	// Allow RSA and ECDSA formats only
	var key crypto.Signer
	switch k := privKey.(type) {
	case *rsa.PrivateKey:
		key = k
	case *ecdsa.PrivateKey:
		key = k
	default:
		return nil, errors.Errorf("the private key file %s is neither in RSA nor ECDSA format", privateKeyPath)
	}

	return key, nil
}


TryLoadCSRAndKeyFromDisk

// TryLoadCSRAndKeyFromDisk tries to load the CSR and key from the disk
func TryLoadCSRAndKeyFromDisk(pkiPath, name string) (*x509.CertificateRequest, crypto.Signer, error) {
	csr, err := TryLoadCSRFromDisk(pkiPath, name)
	if err != nil {
		return nil, nil, errors.Wrap(err, "could not load CSR file")
	}

	key, err := TryLoadKeyFromDisk(pkiPath, name)
	if err != nil {
		return nil, nil, errors.Wrap(err, "could not load key file")
	}

	return csr, key, nil
}


TryLoadPrivatePublicKeyFromDisk

// TryLoadPrivatePublicKeyFromDisk tries to load the key from the disk and validates that it is valid
func TryLoadPrivatePublicKeyFromDisk(pkiPath, name string) (crypto.PrivateKey, crypto.PublicKey, error) {
	privateKeyPath := pathForKey(pkiPath, name)

	// Parse the private key from a file
	privKey, err := keyutil.PrivateKeyFromFile(privateKeyPath)
	if err != nil {
		return nil, nil, errors.Wrapf(err, "couldn't load the private key file %s", privateKeyPath)
	}

	publicKeyPath := pathForPublicKey(pkiPath, name)

	// Parse the public key from a file
	pubKeys, err := keyutil.PublicKeysFromFile(publicKeyPath)
	if err != nil {
		return nil, nil, errors.Wrapf(err, "couldn't load the public key file %s", publicKeyPath)
	}

	// Allow RSA and ECDSA formats only
	switch k := privKey.(type) {
	case *rsa.PrivateKey:
		return k, pubKeys[0].(*rsa.PublicKey), nil
	case *ecdsa.PrivateKey:
		return k, pubKeys[0].(*ecdsa.PublicKey), nil
	default:
		return nil, nil, errors.Errorf("the private key file %s is neither in RSA nor ECDSA format", privateKeyPath)
	}
}


TryLoadCSRFromDisk

// TryLoadCSRFromDisk tries to load the CSR from the disk
func TryLoadCSRFromDisk(pkiPath, name string) (*x509.CertificateRequest, error) {
	csrPath := pathForCSR(pkiPath, name)

	csr, err := CertificateRequestFromFile(csrPath)
	if err != nil {
		return nil, errors.Wrapf(err, "could not load the CSR %s", csrPath)
	}

	return csr, nil
}


PathsForCertAndKey

// PathsForCertAndKey returns the paths for the certificate and key given the path and basename.
func PathsForCertAndKey(pkiPath, name string) (string, string) {
	return pathForCert(pkiPath, name), pathForKey(pkiPath, name)
}


pathForCert

func pathForCert(pkiPath, name string) string {
	return filepath.Join(pkiPath, fmt.Sprintf("%s.crt", name))
}


pathForKey

func pathForKey(pkiPath, name string) string {
	return filepath.Join(pkiPath, fmt.Sprintf("%s.key", name))
}


pathForPublicKey

func pathForPublicKey(pkiPath, name string) string {
	return filepath.Join(pkiPath, fmt.Sprintf("%s.pub", name))
}


pathForCSR

func pathForCSR(pkiPath, name string) string {
	return filepath.Join(pkiPath, fmt.Sprintf("%s.csr", name))
}


GetAPIServerAltNames

// GetAPIServerAltNames builds an AltNames object for to be used when generating apiserver certificate
func GetAPIServerAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames, error) {
	// advertise address
	advertiseAddress := netutils.ParseIPSloppy(cfg.LocalAPIEndpoint.AdvertiseAddress)
	if advertiseAddress == nil {
		return nil, errors.Errorf("error parsing LocalAPIEndpoint AdvertiseAddress %v: is not a valid textual representation of an IP address",
			cfg.LocalAPIEndpoint.AdvertiseAddress)
	}

	internalAPIServerVirtualIP, err := kubeadmconstants.GetAPIServerVirtualIP(cfg.Networking.ServiceSubnet)
	if err != nil {
		return nil, errors.Wrapf(err, "unable to get first IP address from the given CIDR: %v", cfg.Networking.ServiceSubnet)
	}

	// create AltNames with defaults DNSNames/IPs
	altNames := &certutil.AltNames{
		DNSNames: []string{
			cfg.NodeRegistration.Name,
			"kubernetes",
			"kubernetes.default",
			"kubernetes.default.svc",
			fmt.Sprintf("kubernetes.default.svc.%s", cfg.Networking.DNSDomain),
		},
		IPs: []net.IP{
			internalAPIServerVirtualIP,
			advertiseAddress,
		},
	}

	// add cluster controlPlaneEndpoint if present (dns or ip)
	if len(cfg.ControlPlaneEndpoint) > 0 {
		if host, _, err := kubeadmutil.ParseHostPort(cfg.ControlPlaneEndpoint); err == nil {
			if ip := netutils.ParseIPSloppy(host); ip != nil {
				altNames.IPs = append(altNames.IPs, ip)
			} else {
				altNames.DNSNames = append(altNames.DNSNames, host)
			}
		} else {
			return nil, errors.Wrapf(err, "error parsing cluster controlPlaneEndpoint %q", cfg.ControlPlaneEndpoint)
		}
	}

	appendSANsToAltNames(altNames, cfg.APIServer.CertSANs, kubeadmconstants.APIServerCertName)

	return altNames, nil
}


GetEtcdAltNames

// GetEtcdAltNames builds an AltNames object for generating the etcd server certificate.
// `advertise address` and localhost are included in the SAN since this is the interfaces the etcd static pod listens on.
// The user can override the listen address with `Etcd.ExtraArgs` and add SANs with `Etcd.ServerCertSANs`.
func GetEtcdAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames, error) {
	return getAltNames(cfg, kubeadmconstants.EtcdServerCertName)
}


GetEtcdPeerAltNames

// GetEtcdPeerAltNames builds an AltNames object for generating the etcd peer certificate.
// Hostname and `API.AdvertiseAddress` are included if the user chooses to promote the single node etcd cluster into a multi-node one (stacked etcd).
// The user can override the listen address with `Etcd.ExtraArgs` and add SANs with `Etcd.PeerCertSANs`.
func GetEtcdPeerAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames, error) {
	return getAltNames(cfg, kubeadmconstants.EtcdPeerCertName)
}


getAltNames

// getAltNames builds an AltNames object with the cfg and certName.
func getAltNames(cfg *kubeadmapi.InitConfiguration, certName string) (*certutil.AltNames, error) {
	// advertise address
	advertiseAddress := netutils.ParseIPSloppy(cfg.LocalAPIEndpoint.AdvertiseAddress)
	if advertiseAddress == nil {
		return nil, errors.Errorf("error parsing LocalAPIEndpoint AdvertiseAddress %v: is not a valid textual representation of an IP address",
			cfg.LocalAPIEndpoint.AdvertiseAddress)
	}

	// create AltNames with defaults DNSNames/IPs
	altNames := &certutil.AltNames{
		DNSNames: []string{cfg.NodeRegistration.Name, "localhost"},
		IPs:      []net.IP{advertiseAddress, net.IPv4(127, 0, 0, 1), net.IPv6loopback},
	}

	if cfg.Etcd.Local != nil {
		if certName == kubeadmconstants.EtcdServerCertName {
			appendSANsToAltNames(altNames, cfg.Etcd.Local.ServerCertSANs, kubeadmconstants.EtcdServerCertName)
		} else if certName == kubeadmconstants.EtcdPeerCertName {
			appendSANsToAltNames(altNames, cfg.Etcd.Local.PeerCertSANs, kubeadmconstants.EtcdPeerCertName)
		}
	}
	return altNames, nil
}


appendSANsToAltNames

// appendSANsToAltNames parses SANs from as list of strings and adds them to altNames for use on a specific cert
// altNames is passed in with a pointer, and the struct is modified
// valid IP address strings are parsed and added to altNames.IPs as net.IP's
// RFC-1123 compliant DNS strings are added to altNames.DNSNames as strings
// RFC-1123 compliant wildcard DNS strings are added to altNames.DNSNames as strings
// certNames is used to print user facing warnings and should be the name of the cert the altNames will be used for
func appendSANsToAltNames(altNames *certutil.AltNames, SANs []string, certName string) {
	for _, altname := range SANs {
		if ip := netutils.ParseIPSloppy(altname); ip != nil {
			altNames.IPs = append(altNames.IPs, ip)
		} else if len(validation.IsDNS1123Subdomain(altname)) == 0 {
			altNames.DNSNames = append(altNames.DNSNames, altname)
		} else if len(validation.IsWildcardDNS1123Subdomain(altname)) == 0 {
			altNames.DNSNames = append(altNames.DNSNames, altname)
		} else {
			klog.Warningf(
				"[certificates] WARNING: '%s' was not added to the '%s' SAN, because it is not a valid IP or RFC-1123 compliant DNS entry\n",
				altname,
				certName,
			)
		}
	}
}


EncodeCSRPEM

// EncodeCSRPEM returns PEM-encoded CSR data
func EncodeCSRPEM(csr *x509.CertificateRequest) []byte {
	block := pem.Block{
		Type:  certutil.CertificateRequestBlockType,
		Bytes: csr.Raw,
	}
	return pem.EncodeToMemory(&block)
}


parseCSRPEM

func parseCSRPEM(pemCSR []byte) (*x509.CertificateRequest, error) {
	block, _ := pem.Decode(pemCSR)
	if block == nil {
		return nil, errors.New("data doesn't contain a valid certificate request")
	}

	if block.Type != certutil.CertificateRequestBlockType {
		return nil, errors.Errorf("expected block type %q, but PEM had type %q", certutil.CertificateRequestBlockType, block.Type)
	}

	return x509.ParseCertificateRequest(block.Bytes)
}


CertificateRequestFromFile

// CertificateRequestFromFile returns the CertificateRequest from a given PEM-encoded file.
// Returns an error if the file could not be read or if the CSR could not be parsed.
func CertificateRequestFromFile(file string) (*x509.CertificateRequest, error) {
	pemBlock, err := os.ReadFile(file)
	if err != nil {
		return nil, errors.Wrap(err, "failed to read file")
	}

	csr, err := parseCSRPEM(pemBlock)
	if err != nil {
		return nil, errors.Wrapf(err, "error reading certificate request file %s", file)
	}
	return csr, nil
}


NewCSR

// NewCSR creates a new CSR
func NewCSR(cfg CertConfig, key crypto.Signer) (*x509.CertificateRequest, error) {
	RemoveDuplicateAltNames(&cfg.AltNames)

	template := &x509.CertificateRequest{
		Subject: pkix.Name{
			CommonName:   cfg.CommonName,
			Organization: cfg.Organization,
		},
		DNSNames:    cfg.AltNames.DNSNames,
		IPAddresses: cfg.AltNames.IPs,
	}

	csrBytes, err := x509.CreateCertificateRequest(cryptorand.Reader, template, key)

	if err != nil {
		return nil, errors.Wrap(err, "failed to create a CSR")
	}

	return x509.ParseCertificateRequest(csrBytes)
}


EncodeCertPEM

// EncodeCertPEM returns PEM-endcoded certificate data
func EncodeCertPEM(cert *x509.Certificate) []byte {
	block := pem.Block{
		Type:  CertificateBlockType,
		Bytes: cert.Raw,
	}
	return pem.EncodeToMemory(&block)
}


EncodeCertBundlePEM

// EncodeCertBundlePEM returns PEM-endcoded certificate bundle
func EncodeCertBundlePEM(certs []*x509.Certificate) ([]byte, error) {
	buf := bytes.Buffer{}

	block := pem.Block{
		Type: CertificateBlockType,
	}

	for _, cert := range certs {
		block.Bytes = cert.Raw
		if err := pem.Encode(&buf, &block); err != nil {
			return nil, err
		}
	}

	return buf.Bytes(), nil
}


EncodePublicKeyPEM

// EncodePublicKeyPEM returns PEM-encoded public data
func EncodePublicKeyPEM(key crypto.PublicKey) ([]byte, error) {
	der, err := x509.MarshalPKIXPublicKey(key)
	if err != nil {
		return []byte{}, err
	}
	block := pem.Block{
		Type:  PublicKeyBlockType,
		Bytes: der,
	}
	return pem.EncodeToMemory(&block), nil
}


GeneratePrivateKey

func GeneratePrivateKey(keyType x509.PublicKeyAlgorithm) (crypto.Signer, error) {
	if keyType == x509.ECDSA {
		return ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader)
	}

	return rsa.GenerateKey(cryptorand.Reader, rsaKeySize)
}


NewSignedCert

// NewSignedCert creates a signed certificate using the given CA certificate and key
func NewSignedCert(cfg *CertConfig, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer, isCA bool) (*x509.Certificate, error) {
	// returns a uniform random value in [0, max-1), then add 1 to serial to make it a uniform random value in [1, max).
	serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64-1))
	if err != nil {
		return nil, err
	}
	serial = new(big.Int).Add(serial, big.NewInt(1))
	if len(cfg.CommonName) == 0 {
		return nil, errors.New("must specify a CommonName")
	}

	keyUsage := x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature
	if isCA {
		keyUsage |= x509.KeyUsageCertSign
	}

	RemoveDuplicateAltNames(&cfg.AltNames)

	notAfter := time.Now().Add(kubeadmconstants.CertificateValidity).UTC()
	if cfg.NotAfter != nil {
		notAfter = *cfg.NotAfter
	}

	certTmpl := x509.Certificate{
		Subject: pkix.Name{
			CommonName:   cfg.CommonName,
			Organization: cfg.Organization,
		},
		DNSNames:              cfg.AltNames.DNSNames,
		IPAddresses:           cfg.AltNames.IPs,
		SerialNumber:          serial,
		NotBefore:             caCert.NotBefore,
		NotAfter:              notAfter,
		KeyUsage:              keyUsage,
		ExtKeyUsage:           cfg.Usages,
		BasicConstraintsValid: true,
		IsCA:                  isCA,
	}
	certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey)
	if err != nil {
		return nil, err
	}
	return x509.ParseCertificate(certDERBytes)
}


RemoveDuplicateAltNames

// RemoveDuplicateAltNames removes duplicate items in altNames.
func RemoveDuplicateAltNames(altNames *certutil.AltNames) {
	if altNames == nil {
		return
	}

	if altNames.DNSNames != nil {
		altNames.DNSNames = sets.List(sets.New(altNames.DNSNames...))
	}

	ipsKeys := make(map[string]struct{})
	var ips []net.IP
	for _, one := range altNames.IPs {
		if _, ok := ipsKeys[one.String()]; !ok {
			ipsKeys[one.String()] = struct{}{}
			ips = append(ips, one)
		}
	}
	altNames.IPs = ips
}


ValidateCertPeriod

// ValidateCertPeriod checks if the certificate is valid relative to the current time
// (+/- offset)
func ValidateCertPeriod(cert *x509.Certificate, offset time.Duration) error {
	period := fmt.Sprintf("NotBefore: %v, NotAfter: %v", cert.NotBefore, cert.NotAfter)
	now := time.Now().Add(offset)
	if now.Before(cert.NotBefore) {
		return errors.Errorf("the certificate is not valid yet: %s", period)
	}
	if now.After(cert.NotAfter) {
		return errors.Errorf("the certificate has expired: %s", period)
	}
	return nil
}


VerifyCertChain

// VerifyCertChain verifies that a certificate has a valid chain of
// intermediate CAs back to the root CA
func VerifyCertChain(cert *x509.Certificate, intermediates []*x509.Certificate, root *x509.Certificate) error {
	rootPool := x509.NewCertPool()
	rootPool.AddCert(root)

	intermediatePool := x509.NewCertPool()
	for _, c := range intermediates {
		intermediatePool.AddCert(c)
	}

	verifyOptions := x509.VerifyOptions{
		Roots:         rootPool,
		Intermediates: intermediatePool,
		KeyUsages:     []x509.ExtKeyUsage{x509.ExtKeyUsageAny},
	}

	if _, err := cert.Verify(verifyOptions); err != nil {
		return err
	}

	return nil
}


NewSet

// NewSet returns a new, empty PubKeyPinSet
func NewSet() *Set {
	return &Set{make(map[string]bool)}
}


Allow

// Allow adds an allowed public key hash to the Set
func (s *Set) Allow(pubKeyHashes ...string) error {
	for _, pubKeyHash := range pubKeyHashes {
		parts := strings.Split(pubKeyHash, ":")
		if len(parts) != 2 {
			return errors.Errorf("invalid hash, expected \"format:hex-value\". "+
				"Known format(s) are: %s", supportedFormats)
		}
		format, value := parts[0], parts[1]

		switch strings.ToLower(format) {
		case "sha256":
			if err := s.allowSHA256(value); err != nil {
				return errors.Errorf("invalid hash %q, %v", pubKeyHash, err)
			}
		default:
			return errors.Errorf("unknown hash format %q. Known format(s) are: %s", format, supportedFormats)
		}
	}
	return nil
}


CheckAny

// CheckAny checks if at least one certificate matches one of the public keys in the set
func (s *Set) CheckAny(certificates []*x509.Certificate) error {
	var hashes []string

	for _, certificate := range certificates {
		if s.checkSHA256(certificate) {
			return nil
		}

		hashes = append(hashes, Hash(certificate))
	}
	return errors.Errorf("none of the public keys %q are pinned", strings.Join(hashes, ":"))
}


Empty

// Empty returns true if the Set contains no pinned public keys.
func (s *Set) Empty() bool {
	return len(s.sha256Hashes) == 0
}


Hash

// Hash calculates the SHA-256 hash of the Subject Public Key Information (SPKI)
// object in an x509 certificate (in DER encoding). It returns the full hash as a
// hex encoded string (suitable for passing to Set.Allow).
func Hash(certificate *x509.Certificate) string {
	spkiHash := sha256.Sum256(certificate.RawSubjectPublicKeyInfo)
	return formatSHA256 + ":" + strings.ToLower(hex.EncodeToString(spkiHash[:]))
}


// allowSHA256 validates a "sha256" format hash and adds a canonical version of it into the Set
func (s *Set) allowSHA256(hash string) error {
	// validate that the hash is the right length to be a full SHA-256 hash
	hashLength := hex.DecodedLen(len(hash))
	if hashLength != sha256.Size {
		return errors.Errorf("expected a %d byte SHA-256 hash, found %d bytes", sha256.Size, hashLength)
	}

	// validate that the hash is valid hex
	_, err := hex.DecodeString(hash)
	if err != nil {
		return errors.Wrap(err, "could not decode SHA-256 from hex")
	}

	// in the end, just store the original hex string in memory (in lowercase)
	s.sha256Hashes[strings.ToLower(hash)] = true
	return nil
}


// checkSHA256 returns true if the certificate's "sha256" hash is pinned in the Set
func (s *Set) checkSHA256(certificate *x509.Certificate) bool {
	actualHash := sha256.Sum256(certificate.RawSubjectPublicKeyInfo)
	actualHashHex := strings.ToLower(hex.EncodeToString(actualHash[:]))
	return s.sha256Hashes[actualHashHex]
}


NewContainerRuntime

// NewContainerRuntime sets up and returns a ContainerRuntime struct
func NewContainerRuntime(execer utilsexec.Interface, criSocket string) (ContainerRuntime, error) {
	const toolName = "crictl"
	crictlPath, err := execer.LookPath(toolName)
	if err != nil {
		return nil, errors.Wrapf(err, "%s is required by the container runtime", toolName)
	}
	return &CRIRuntime{execer, criSocket, crictlPath}, nil
}


Socket

// Socket returns the CRI socket endpoint
func (runtime *CRIRuntime) Socket() string {
	return runtime.criSocket
}


crictl

// crictl creates a crictl command for the provided args.
func (runtime *CRIRuntime) crictl(args ...string) utilsexec.Cmd {
	cmd := runtime.exec.Command(runtime.crictlPath, append([]string{"-r", runtime.Socket()}, args...)...)
	cmd.SetEnv(os.Environ())
	return cmd
}


IsRunning

// IsRunning checks if runtime is running
func (runtime *CRIRuntime) IsRunning() error {
	if out, err := runtime.crictl("info").CombinedOutput(); err != nil {
		return errors.Wrapf(err, "container runtime is not running: output: %s, error", string(out))
	}
	return nil
}


ListKubeContainers

// ListKubeContainers lists running k8s CRI pods
func (runtime *CRIRuntime) ListKubeContainers() ([]string, error) {
	// Disable debug mode regardless how the crictl is configured so that the debug info won't be
	// iterpreted to the Pod ID.
	args := []string{"-D=false", "pods", "-q"}
	out, err := runtime.crictl(args...).CombinedOutput()
	if err != nil {
		return nil, errors.Wrapf(err, "output: %s, error", string(out))
	}
	pods := []string{}
	pods = append(pods, strings.Fields(string(out))...)
	return pods, nil
}


RemoveContainers

// RemoveContainers removes running k8s pods
func (runtime *CRIRuntime) RemoveContainers(containers []string) error {
	errs := []error{}
	for _, container := range containers {
		var lastErr error
		for i := 0; i < constants.RemoveContainerRetry; i++ {
			klog.V(5).Infof("Attempting to remove container %v", container)
			out, err := runtime.crictl("stopp", container).CombinedOutput()
			if err != nil {
				lastErr = errors.Wrapf(err, "failed to stop running pod %s: output: %s", container, string(out))
				continue
			}
			out, err = runtime.crictl("rmp", container).CombinedOutput()
			if err != nil {
				lastErr = errors.Wrapf(err, "failed to remove running container %s: output: %s", container, string(out))
				continue
			}
			lastErr = nil
			break
		}

		if lastErr != nil {
			errs = append(errs, lastErr)
		}
	}
	return errorsutil.NewAggregate(errs)
}


PullImage

// PullImage pulls the image
func (runtime *CRIRuntime) PullImage(image string) error {
	var err error
	var out []byte
	for i := 0; i < constants.PullImageRetry; i++ {
		out, err = runtime.crictl("pull", image).CombinedOutput()
		if err == nil {
			return nil
		}
	}
	return errors.Wrapf(err, "output: %s, error", out)
}


ImageExists

// ImageExists checks to see if the image exists on the system
func (runtime *CRIRuntime) ImageExists(image string) (bool, error) {
	err := runtime.crictl("inspecti", image).Run()
	return err == nil, nil
}


detectCRISocketImpl

// detectCRISocketImpl is separated out only for test purposes, DON'T call it directly, use DetectCRISocket instead
func detectCRISocketImpl(isSocket func(string) bool, knownCRISockets []string) (string, error) {
	foundCRISockets := []string{}

	for _, socket := range knownCRISockets {
		if isSocket(socket) {
			foundCRISockets = append(foundCRISockets, socket)
		}
	}

	switch len(foundCRISockets) {
	case 0:
		// Fall back to the default socket if no CRI is detected, we can error out later on if we need it
		return constants.DefaultCRISocket, nil
	case 1:
		// Precisely one CRI found, use that
		return foundCRISockets[0], nil
	default:
		// Multiple CRIs installed?
		return "", errors.Errorf("Found multiple CRI endpoints on the host. Please define which one do you wish "+
			"to use by setting the 'criSocket' field in the kubeadm configuration file: %s",
			strings.Join(foundCRISockets, ", "))
	}
}


DetectCRISocket

// DetectCRISocket uses a list of known CRI sockets to detect one. If more than one or none is discovered, an error is returned.
func DetectCRISocket() (string, error) {
	return detectCRISocketImpl(isExistingSocket, defaultKnownCRISockets)
}


SandboxImage

// SandboxImage returns the sandbox image used by the container runtime
func (runtime *CRIRuntime) SandboxImage() (string, error) {
	args := []string{"-D=false", "info", "-o", "go-template", "--template", "{{.config.sandboxImage}}"}
	out, err := runtime.crictl(args...).CombinedOutput()
	if err != nil {
		return "", errors.Wrapf(err, "output: %s, error", string(out))
	}

	sandboxImage := strings.TrimSpace(string(out))
	if len(sandboxImage) > 0 {
		return sandboxImage, nil
	}

	return "", errors.Errorf("the detected sandbox image is empty")
}


isExistingSocket

// isExistingSocket checks if path exists and is domain socket
func isExistingSocket(path string) bool {
	u, err := url.Parse(path)
	if err != nil {
		// should not happen, since we are trying to access known / hardcoded sockets
		return false
	}

	c, err := net.Dial(u.Scheme, u.Path)
	if err != nil {
		return false
	}
	defer c.Close()
	return true
}


// isExistingSocket checks if path exists and is domain socket
func isExistingSocket(path string) bool {
	u, err := url.Parse(path)
	if err != nil {
		// should not happen, since we are trying to access known / hardcoded sockets
		return false
	}

	// the dial path must be without "npipe://"
	_, err = winio.DialPipe(u.Path, nil)
	if err != nil {
		return false
	}

	return true
}


ComponentPod

// ComponentPod returns a Pod object from the container, volume and annotations specifications
func ComponentPod(container v1.Container, volumes map[string]v1.Volume, annotations map[string]string) v1.Pod {
	// priority value for system-node-critical class
	priority := int32(2000001000)
	return v1.Pod{
		TypeMeta: metav1.TypeMeta{
			APIVersion: "v1",
			Kind:       "Pod",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      container.Name,
			Namespace: metav1.NamespaceSystem,
			// The component and tier labels are useful for quickly identifying the control plane Pods when doing a .List()
			// against Pods in the kube-system namespace. Can for example be used together with the WaitForPodsWithLabel function
			Labels:      map[string]string{"component": container.Name, "tier": kubeadmconstants.ControlPlaneTier},
			Annotations: annotations,
		},
		Spec: v1.PodSpec{
			Containers:        []v1.Container{container},
			Priority:          &priority,
			PriorityClassName: "system-node-critical",
			HostNetwork:       true,
			Volumes:           VolumeMapToSlice(volumes),
			SecurityContext: &v1.PodSecurityContext{
				SeccompProfile: &v1.SeccompProfile{
					Type: v1.SeccompProfileTypeRuntimeDefault,
				},
			},
		},
	}
}


ComponentResources

// ComponentResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU
func ComponentResources(cpu string) v1.ResourceRequirements {
	return v1.ResourceRequirements{
		Requests: v1.ResourceList{
			v1.ResourceName(v1.ResourceCPU): resource.MustParse(cpu),
		},
	}
}


NewVolume

// NewVolume creates a v1.Volume with a hostPath mount to the specified location
func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume {
	return v1.Volume{
		Name: name,
		VolumeSource: v1.VolumeSource{
			HostPath: &v1.HostPathVolumeSource{
				Path: path,
				Type: pathType,
			},
		},
	}
}


NewVolumeMount

// NewVolumeMount creates a v1.VolumeMount to the specified location
func NewVolumeMount(name, path string, readOnly bool) v1.VolumeMount {
	return v1.VolumeMount{
		Name:      name,
		MountPath: path,
		ReadOnly:  readOnly,
	}
}


VolumeMapToSlice

// VolumeMapToSlice returns a slice of volumes from a map's values
func VolumeMapToSlice(volumes map[string]v1.Volume) []v1.Volume {
	v := make([]v1.Volume, 0, len(volumes))

	for _, vol := range volumes {
		v = append(v, vol)
	}

	sort.Slice(v, func(i, j int) bool {
		return strings.Compare(v[i].Name, v[j].Name) == -1
	})

	return v
}


VolumeMountMapToSlice

// VolumeMountMapToSlice returns a slice of volumes from a map's values
func VolumeMountMapToSlice(volumeMounts map[string]v1.VolumeMount) []v1.VolumeMount {
	v := make([]v1.VolumeMount, 0, len(volumeMounts))

	for _, volMount := range volumeMounts {
		v = append(v, volMount)
	}

	sort.Slice(v, func(i, j int) bool {
		return strings.Compare(v[i].Name, v[j].Name) == -1
	})

	return v
}


GetExtraParameters

// GetExtraParameters builds a list of flag arguments two string-string maps, one with default, base commands and one with overrides
func GetExtraParameters(overrides map[string]string, defaults map[string]string) []string {
	var command []string
	for k, v := range overrides {
		if len(v) > 0 {
			command = append(command, fmt.Sprintf("--%s=%s", k, v))
		}
	}
	for k, v := range defaults {
		if _, overrideExists := overrides[k]; !overrideExists {
			command = append(command, fmt.Sprintf("--%s=%s", k, v))
		}
	}
	return command
}


PatchStaticPod

// PatchStaticPod applies patches stored in patchesDir to a static Pod.
func PatchStaticPod(pod *v1.Pod, patchesDir string, output io.Writer) (*v1.Pod, error) {
	// Marshal the Pod manifest into YAML.
	podYAML, err := kubeadmutil.MarshalToYaml(pod, v1.SchemeGroupVersion)
	if err != nil {
		return pod, errors.Wrapf(err, "failed to marshal Pod manifest to YAML")
	}

	patchManager, err := patches.GetPatchManagerForPath(patchesDir, patches.KnownTargets(), output)
	if err != nil {
		return pod, err
	}

	patchTarget := &patches.PatchTarget{
		Name:                      pod.Name,
		StrategicMergePatchObject: v1.Pod{},
		Data:                      podYAML,
	}
	if err := patchManager.ApplyPatchesToTarget(patchTarget); err != nil {
		return pod, err
	}

	obj, err := kubeadmutil.UnmarshalFromYaml(patchTarget.Data, v1.SchemeGroupVersion)
	if err != nil {
		return pod, errors.Wrap(err, "failed to unmarshal patched manifest from YAML")
	}

	pod2, ok := obj.(*v1.Pod)
	if !ok {
		return pod, errors.Wrap(err, "patched manifest is not a valid Pod object")
	}

	return pod2, nil
}


WriteStaticPodToDisk

// WriteStaticPodToDisk writes a static pod file to disk
func WriteStaticPodToDisk(componentName, manifestDir string, pod v1.Pod) error {

	// creates target folder if not already exists
	if err := os.MkdirAll(manifestDir, 0700); err != nil {
		return errors.Wrapf(err, "failed to create directory %q", manifestDir)
	}

	// writes the pod to disk
	serialized, err := kubeadmutil.MarshalToYaml(&pod, v1.SchemeGroupVersion)
	if err != nil {
		return errors.Wrapf(err, "failed to marshal manifest for %q to YAML", componentName)
	}

	filename := kubeadmconstants.GetStaticPodFilepath(componentName, manifestDir)

	if err := os.WriteFile(filename, serialized, 0600); err != nil {
		return errors.Wrapf(err, "failed to write static pod manifest file for %q (%q)", componentName, filename)
	}

	return nil
}


ReadStaticPodFromDisk

// ReadStaticPodFromDisk reads a static pod file from disk
func ReadStaticPodFromDisk(manifestPath string) (*v1.Pod, error) {
	buf, err := os.ReadFile(manifestPath)
	if err != nil {
		return &v1.Pod{}, errors.Wrapf(err, "failed to read manifest for %q", manifestPath)
	}

	obj, err := kubeadmutil.UnmarshalFromYaml(buf, v1.SchemeGroupVersion)
	if err != nil {
		return &v1.Pod{}, errors.Errorf("failed to unmarshal manifest for %q from YAML: %v", manifestPath, err)
	}

	pod := obj.(*v1.Pod)

	return pod, nil
}


LivenessProbe

// LivenessProbe creates a Probe object with a HTTPGet handler
func LivenessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe {
	// sets initialDelaySeconds same as periodSeconds to skip one period before running a check
	return createHTTPProbe(host, path, port, scheme, 10, 15, 8, 10)
}


ReadinessProbe

// ReadinessProbe creates a Probe object with a HTTPGet handler
func ReadinessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe {
	// sets initialDelaySeconds as '0' because we don't want to delay user infrastructure checks
	// looking for "ready" status on kubeadm static Pods
	return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 1)
}


StartupProbe

// StartupProbe creates a Probe object with a HTTPGet handler
func StartupProbe(host, path string, port int, scheme v1.URIScheme, timeoutForControlPlane *metav1.Duration) *v1.Probe {
	periodSeconds, timeoutForControlPlaneSeconds := int32(10), kubeadmconstants.DefaultControlPlaneTimeout.Seconds()
	if timeoutForControlPlane != nil {
		timeoutForControlPlaneSeconds = timeoutForControlPlane.Seconds()
	}
	// sets failureThreshold big enough to guarantee the full timeout can cover the worst case scenario for the control-plane to come alive
	// we ignore initialDelaySeconds in the calculation here for simplicity
	failureThreshold := int32(math.Ceil(timeoutForControlPlaneSeconds / float64(periodSeconds)))
	// sets initialDelaySeconds same as periodSeconds to skip one period before running a check
	return createHTTPProbe(host, path, port, scheme, periodSeconds, 15, failureThreshold, periodSeconds)
}


createHTTPProbe

func createHTTPProbe(host, path string, port int, scheme v1.URIScheme, initialDelaySeconds, timeoutSeconds, failureThreshold, periodSeconds int32) *v1.Probe {
	return &v1.Probe{
		ProbeHandler: v1.ProbeHandler{
			HTTPGet: &v1.HTTPGetAction{
				Host:   host,
				Path:   path,
				Port:   intstr.FromInt(port),
				Scheme: scheme,
			},
		},
		InitialDelaySeconds: initialDelaySeconds,
		TimeoutSeconds:      timeoutSeconds,
		FailureThreshold:    failureThreshold,
		PeriodSeconds:       periodSeconds,
	}
}


GetAPIServerProbeAddress

// GetAPIServerProbeAddress returns the probe address for the API server
func GetAPIServerProbeAddress(endpoint *kubeadmapi.APIEndpoint) string {
	if endpoint != nil && endpoint.AdvertiseAddress != "" {
		return getProbeAddress(endpoint.AdvertiseAddress)
	}

	return "127.0.0.1"
}


GetControllerManagerProbeAddress

// GetControllerManagerProbeAddress returns the kubernetes controller manager probe address
func GetControllerManagerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) string {
	if addr, exists := cfg.ControllerManager.ExtraArgs[kubeControllerManagerBindAddressArg]; exists {
		return getProbeAddress(addr)
	}
	return "127.0.0.1"
}


GetSchedulerProbeAddress

// GetSchedulerProbeAddress returns the kubernetes scheduler probe address
func GetSchedulerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) string {
	if addr, exists := cfg.Scheduler.ExtraArgs[kubeSchedulerBindAddressArg]; exists {
		return getProbeAddress(addr)
	}
	return "127.0.0.1"
}


GetEtcdProbeEndpoint

// GetEtcdProbeEndpoint takes a kubeadm Etcd configuration object and attempts to parse
// the first URL in the listen-metrics-urls argument, returning an etcd probe hostname,
// port and scheme
func GetEtcdProbeEndpoint(cfg *kubeadmapi.Etcd, isIPv6 bool) (string, int, v1.URIScheme) {
	localhost := "127.0.0.1"
	if isIPv6 {
		localhost = "::1"
	}
	if cfg.Local == nil || cfg.Local.ExtraArgs == nil {
		return localhost, kubeadmconstants.EtcdMetricsPort, v1.URISchemeHTTP
	}
	if arg, exists := cfg.Local.ExtraArgs["listen-metrics-urls"]; exists {
		// Use the first url in the listen-metrics-urls if multiple URL's are specified.
		arg = strings.Split(arg, ",")[0]
		parsedURL, err := url.Parse(arg)
		if err != nil {
			return localhost, kubeadmconstants.EtcdMetricsPort, v1.URISchemeHTTP
		}
		// Parse scheme
		scheme := v1.URISchemeHTTP
		if parsedURL.Scheme == "https" {
			scheme = v1.URISchemeHTTPS
		}
		// Parse hostname
		hostname := parsedURL.Hostname()
		if len(hostname) == 0 {
			hostname = localhost
		}
		// Parse port
		port := kubeadmconstants.EtcdMetricsPort
		portStr := parsedURL.Port()
		if len(portStr) != 0 {
			p, err := kubeadmutil.ParsePort(portStr)
			if err == nil {
				port = p
			}
		}
		return hostname, port, scheme
	}
	return localhost, kubeadmconstants.EtcdMetricsPort, v1.URISchemeHTTP
}


ManifestFilesAreEqual

// ManifestFilesAreEqual compares 2 files. It returns true if their contents are equal, false otherwise
func ManifestFilesAreEqual(path1, path2 string) (bool, error) {
	pod1, err := ReadStaticPodFromDisk(path1)
	if err != nil {
		return false, err
	}
	pod2, err := ReadStaticPodFromDisk(path2)
	if err != nil {
		return false, err
	}

	hasher := md5.New()
	DeepHashObject(hasher, pod1)
	hash1 := hasher.Sum(nil)[0:]
	DeepHashObject(hasher, pod2)
	hash2 := hasher.Sum(nil)[0:]

	return bytes.Equal(hash1, hash2), nil
}


getProbeAddress

// getProbeAddress returns a valid probe address.
// Kubeadm uses the bind-address to configure the probe address. It's common to use the
// unspecified address "0.0.0.0" or "::" as bind-address when we want to listen in all interfaces,
// however this address can't be used as probe #86504.
// If the address is an unspecified address getProbeAddress returns empty,
// that means that kubelet will use the PodIP as probe address.
func getProbeAddress(addr string) string {
	if addr == "0.0.0.0" || addr == "::" {
		return ""
	}
	return addr
}


GetUsersAndGroups

func GetUsersAndGroups() (*users.UsersAndGroups, error) {
	var err error
	usersAndGroupsOnce.Do(func() {
		usersAndGroups, err = users.AddUsersAndGroups()
	})
	return usersAndGroups, err
}


DeepHashObject

// DeepHashObject writes specified object to hash using the spew library
// which follows pointers and prints actual values of the nested objects
// ensuring the hash does not change when a pointer changes.
// Copied from k8s.io/kubernetes/pkg/util/hash/hash.go#DeepHashObject
func DeepHashObject(hasher hash.Hash, objectToWrite interface{}) {
	hasher.Reset()
	printer := spew.ConfigState{
		Indent:         " ",
		SortKeys:       true,
		DisableMethods: true,
		SpewKeys:       true,
	}
	printer.Fprintf(hasher, "%#v", objectToWrite)
}


RunComponentAsNonRoot

// RunComponentAsNonRoot updates the pod manifest and the hostVolume permissions to run as non root.
func RunComponentAsNonRoot(componentName string, pod *v1.Pod, usersAndGroups *users.UsersAndGroups, cfg *kubeadmapi.ClusterConfiguration) error {
	switch componentName {
	case kubeadmconstants.KubeAPIServer:
		return runKubeAPIServerAsNonRoot(
			pod,
			usersAndGroups.Users.ID(kubeadmconstants.KubeAPIServerUserName),
			usersAndGroups.Groups.ID(kubeadmconstants.KubeAPIServerUserName),
			usersAndGroups.Groups.ID(kubeadmconstants.ServiceAccountKeyReadersGroupName),
			users.UpdatePathOwnerAndPermissions,
			cfg,
		)
	case kubeadmconstants.KubeControllerManager:
		return runKubeControllerManagerAsNonRoot(
			pod,
			usersAndGroups.Users.ID(kubeadmconstants.KubeControllerManagerUserName),
			usersAndGroups.Groups.ID(kubeadmconstants.KubeControllerManagerUserName),
			usersAndGroups.Groups.ID(kubeadmconstants.ServiceAccountKeyReadersGroupName),
			users.UpdatePathOwnerAndPermissions,
			cfg,
		)
	case kubeadmconstants.KubeScheduler:
		return runKubeSchedulerAsNonRoot(
			pod,
			usersAndGroups.Users.ID(kubeadmconstants.KubeSchedulerUserName),
			usersAndGroups.Groups.ID(kubeadmconstants.KubeSchedulerUserName),
			users.UpdatePathOwnerAndPermissions,
		)
	case kubeadmconstants.Etcd:
		return runEtcdAsNonRoot(
			pod,
			usersAndGroups.Users.ID(kubeadmconstants.EtcdUserName),
			usersAndGroups.Groups.ID(kubeadmconstants.EtcdUserName),
			users.UpdatePathOwnerAndPermissions,
			users.UpdatePathOwner,
			cfg,
		)
	}
	return errors.New(fmt.Sprintf("component name %q is not valid", componentName))
}


runKubeAPIServerAsNonRoot

// runKubeAPIServerAsNonRoot updates the pod manifest and the hostVolume permissions to run kube-apiserver as non root.
func runKubeAPIServerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error {
	saPublicKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName)
	if err := updatePathOwnerAndPermissions(saPublicKeyFile, *runAsUser, *runAsGroup, 0600); err != nil {
		return err
	}
	saPrivateKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPrivateKeyName)
	if err := updatePathOwnerAndPermissions(saPrivateKeyFile, 0, *supplementalGroup, 0640); err != nil {
		return err
	}
	apiServerKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKeyName)
	if err := updatePathOwnerAndPermissions(apiServerKeyFile, *runAsUser, *runAsGroup, 0600); err != nil {
		return err
	}
	apiServerKubeletClientKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientKeyName)
	if err := updatePathOwnerAndPermissions(apiServerKubeletClientKeyFile, *runAsUser, *runAsGroup, 0600); err != nil {
		return err
	}
	frontProxyClientKeyName := filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName)
	if err := updatePathOwnerAndPermissions(frontProxyClientKeyName, *runAsUser, *runAsGroup, 0600); err != nil {
		return err
	}
	if cfg.Etcd.External == nil {
		apiServerEtcdClientKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientKeyName)
		if err := updatePathOwnerAndPermissions(apiServerEtcdClientKeyFile, *runAsUser, *runAsGroup, 0600); err != nil {
			return err
		}
	}
	pod.Spec.Containers[0].SecurityContext = &v1.SecurityContext{
		Capabilities: &v1.Capabilities{
			// We drop all capabilities that are added by default.
			Drop: []v1.Capability{"ALL"},
			// kube-apiserver binary has the file capability cap_net_bind_service applied to it.
			// This means that we must add this capability when running as non-root even if the
			// capability is not required.
			Add: []v1.Capability{"NET_BIND_SERVICE"},
		},
	}
	pod.Spec.SecurityContext.RunAsGroup = runAsGroup
	pod.Spec.SecurityContext.RunAsUser = runAsUser
	pod.Spec.SecurityContext.SupplementalGroups = []int64{*supplementalGroup}
	return nil
}


runKubeControllerManagerAsNonRoot

// runKubeControllerManagerAsNonRoot updates the pod manifest and the hostVolume permissions to run kube-controller-manager as non root.
func runKubeControllerManagerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error {
	kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName)
	if err := updatePathOwnerAndPermissions(kubeconfigFile, *runAsUser, *runAsGroup, 0600); err != nil {
		return err
	}
	saPrivateKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPrivateKeyName)
	if err := updatePathOwnerAndPermissions(saPrivateKeyFile, 0, *supplementalGroup, 0640); err != nil {
		return err
	}
	if res, _ := certphase.UsingExternalCA(cfg); !res {
		caKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.CAKeyName)
		err := updatePathOwnerAndPermissions(caKeyFile, *runAsUser, *runAsGroup, 0600)
		if err != nil {
			return err
		}
	}
	pod.Spec.Containers[0].SecurityContext = &v1.SecurityContext{
		AllowPrivilegeEscalation: pointer.Bool(false),
		Capabilities: &v1.Capabilities{
			// We drop all capabilities that are added by default.
			Drop: []v1.Capability{"ALL"},
		},
	}
	pod.Spec.SecurityContext.RunAsUser = runAsUser
	pod.Spec.SecurityContext.RunAsGroup = runAsGroup
	pod.Spec.SecurityContext.SupplementalGroups = []int64{*supplementalGroup}
	return nil
}


runKubeSchedulerAsNonRoot

// runKubeSchedulerAsNonRoot updates the pod manifest and the hostVolume permissions to run kube-scheduler as non root.
func runKubeSchedulerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc) error {
	kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName)
	if err := updatePathOwnerAndPermissions(kubeconfigFile, *runAsUser, *runAsGroup, 0600); err != nil {
		return err
	}
	pod.Spec.Containers[0].SecurityContext = &v1.SecurityContext{
		AllowPrivilegeEscalation: pointer.Bool(false),
		// We drop all capabilities that are added by default.
		Capabilities: &v1.Capabilities{
			Drop: []v1.Capability{"ALL"},
		},
	}
	pod.Spec.SecurityContext.RunAsUser = runAsUser
	pod.Spec.SecurityContext.RunAsGroup = runAsGroup
	return nil
}


runEtcdAsNonRoot

// runEtcdAsNonRoot updates the pod manifest and the hostVolume permissions to run etcd as non root.
func runEtcdAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc, updatePathOwner pathOwnerUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error {
	if err := updatePathOwner(cfg.Etcd.Local.DataDir, *runAsUser, *runAsGroup); err != nil {
		return err
	}
	etcdServerKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName)
	if err := updatePathOwnerAndPermissions(etcdServerKeyFile, *runAsUser, *runAsGroup, 0600); err != nil {
		return err
	}
	etcdPeerKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName)
	if err := updatePathOwnerAndPermissions(etcdPeerKeyFile, *runAsUser, *runAsGroup, 0600); err != nil {
		return err
	}
	pod.Spec.Containers[0].SecurityContext = &v1.SecurityContext{
		AllowPrivilegeEscalation: pointer.Bool(false),
		// We drop all capabilities that are added by default.
		Capabilities: &v1.Capabilities{
			Drop: []v1.Capability{"ALL"},
		},
	}
	pod.Spec.SecurityContext.RunAsUser = runAsUser
	pod.Spec.SecurityContext.RunAsGroup = runAsGroup
	return nil
}


RunComponentAsNonRoot

// RunComponentAsNonRoot is a NO-OP on non linux.
func RunComponentAsNonRoot(componentName string, pod *v1.Pod, usersAndGroups *users.UsersAndGroups, cfg *kubeadmapi.ClusterConfiguration) error {
	return nil
}


ParseTemplate

// ParseTemplate validates and parses passed as argument template
func ParseTemplate(strtmpl string, obj interface{}) ([]byte, error) {
	var buf bytes.Buffer
	tmpl, err := template.New("template").Parse(strtmpl)
	if err != nil {
		return nil, errors.Wrap(err, "error when parsing template")
	}
	err = tmpl.Execute(&buf, obj)
	if err != nil {
		return nil, errors.Wrap(err, "error when executing template")
	}
	return buf.Bytes(), nil
}


ID

// ID returns the ID for an entry based on the entry name.
// In case of a user entry it returns the user UID.
// In case of a group entry it returns the group GID.
// It returns nil if no such entry exists.
func (u *EntryMap) ID(name string) *int64 {
	entry, ok := u.entries[name]
	if !ok {
		return nil
	}
	id := entry.id
	return &id
}


String

// String converts an EntryMap object to a readable string.
func (u *EntryMap) String() string {
	lines := make([]string, 0, len(u.entries))
	for k, e := range u.entries {
		lines = append(lines, fmt.Sprintf("%s{%d,%d};", k, e.id, e.gid))
	}
	sort.Strings(lines)
	return strings.Join(lines, "")
}


AddUsersAndGroups

// Is a public wrapper around addUsersAndGroupsImpl with default system file paths.
func AddUsersAndGroups() (*UsersAndGroups, error) {
	return addUsersAndGroupsImpl(fileEtcLoginDefs, fileEtcPasswd, fileEtcGroup)
}


addUsersAndGroupsImpl

// addUsersAndGroupsImpl adds the managed users and groups to the files specified
// by pathUsers and pathGroups. It uses the file specified with pathLoginDef to
// determine limits for UID and GID. If managed users and groups exist in these files
// validation is performed on them. The function returns a pointer to a Users object
// that can be used to return UID and GID of managed users.
func addUsersAndGroupsImpl(pathLoginDef, pathUsers, pathGroups string) (*UsersAndGroups, error) {
	klog.V(1).Info("Adding managed users and groups")
	klog.V(klogLevel).Infof("Parsing %q", pathLoginDef)

	// Read and parse /etc/login.def. Some distributions might be missing this file, which makes
	// them non-standard. If an error occurs fallback to defaults by passing an empty string
	// to parseLoginDefs().
	var loginDef string
	f, close, err := openFileWithLock(pathLoginDef)
	if err != nil {
		klog.V(1).Infof("Could not open %q, using default system limits: %v", pathLoginDef, err)
	} else {
		loginDef, err = readFile(f)
		if err != nil {
			klog.V(1).Infof("Could not read %q, using default system limits: %v", pathLoginDef, err)
		}
		close()
	}
	limits, err := parseLoginDefs(loginDef)
	if err != nil {
		return nil, err
	}

	klog.V(klogLevel).Infof("Using system UID/GID limits: %+v", limits)
	klog.V(klogLevel).Infof("Parsing %q and %q", pathUsers, pathGroups)

	// Open /etc/passwd and /etc/group with locks.
	fUsers, close, err := openFileWithLock(pathUsers)
	if err != nil {
		return nil, err
	}
	defer close()
	fGroups, close, err := openFileWithLock(pathGroups)
	if err != nil {
		return nil, err
	}
	defer close()

	// Read the files.
	fileUsers, err := readFile(fUsers)
	if err != nil {
		return nil, err
	}
	fileGroups, err := readFile(fGroups)
	if err != nil {
		return nil, err
	}

	// Parse the files.
	users, err := parseEntries(fileUsers, totalFieldsUser)
	if err != nil {
		return nil, errors.Wrapf(err, "could not parse %q", pathUsers)
	}
	groups, err := parseEntries(fileGroups, totalFieldsGroup)
	if err != nil {
		return nil, errors.Wrapf(err, "could not parse %q", pathGroups)
	}

	klog.V(klogLevel).Info("Validating existing users and groups")

	// Validate for existing tracked entries based on limits.
	usersToCreate, groupsToCreate, err := validateEntries(users, groups, limits)
	if err != nil {
		return nil, errors.Wrap(err, "error validating existing users and groups")
	}

	// Allocate and assign IDs to users / groups.
	allocUIDs, err := allocateIDs(users, limits.minUID, limits.maxUID, len(usersToCreate))
	if err != nil {
		return nil, err
	}
	allocGIDs, err := allocateIDs(groups, limits.minGID, limits.maxGID, len(groupsToCreate))
	if err != nil {
		return nil, err
	}
	if err := assignUserAndGroupIDs(groups, usersToCreate, groupsToCreate, allocUIDs, allocGIDs); err != nil {
		return nil, err
	}

	if len(usersToCreate) > 0 {
		klog.V(klogLevel).Infof("Adding users: %s", entriesToString(usersToCreate))
	}
	if len(groupsToCreate) > 0 {
		klog.V(klogLevel).Infof("Adding groups: %s", entriesToString(groupsToCreate))
	}

	// Add users and groups.
	fileUsers = addEntries(fileUsers, usersToCreate, createUser)
	fileGroups = addEntries(fileGroups, groupsToCreate, createGroup)

	// Write the files.
	klog.V(klogLevel).Infof("Writing %q and %q", pathUsers, pathGroups)
	if err := writeFile(fUsers, fileUsers); err != nil {
		return nil, err
	}
	if err := writeFile(fGroups, fileGroups); err != nil {
		return nil, err
	}

	// Prepare the maps of users and groups.
	usersConcat := append(users, usersToCreate...)
	mapUsers, err := entriesToEntryMap(usersConcat, usersToCreateSpec)
	if err != nil {
		return nil, err
	}
	groupsConcat := append(groups, groupsToCreate...)
	mapGroups, err := entriesToEntryMap(groupsConcat, groupsToCreateSpec)
	if err != nil {
		return nil, err
	}
	return &UsersAndGroups{Users: mapUsers, Groups: mapGroups}, nil
}


RemoveUsersAndGroups

// RemoveUsersAndGroups is a public wrapper around removeUsersAndGroupsImpl with
// default system file paths.
func RemoveUsersAndGroups() error {
	return removeUsersAndGroupsImpl(fileEtcPasswd, fileEtcGroup)
}


removeUsersAndGroupsImpl

// removeUsersAndGroupsImpl removes the managed users and groups from the files specified
// by pathUsers and pathGroups.
func removeUsersAndGroupsImpl(pathUsers, pathGroups string) error {
	klog.V(1).Info("Removing managed users and groups")
	klog.V(klogLevel).Infof("Opening %q and %q", pathUsers, pathGroups)

	// Open /etc/passwd and /etc/group.
	fUsers, close, err := openFileWithLock(pathUsers)
	if err != nil {
		return err
	}
	defer close()
	fGroups, close, err := openFileWithLock(pathGroups)
	if err != nil {
		return err
	}
	defer close()

	// Read the files.
	fileUsers, err := readFile(fUsers)
	if err != nil {
		return err
	}
	fileGroups, err := readFile(fGroups)
	if err != nil {
		return err
	}

	klog.V(klogLevel).Infof("Removing users: %s", entriesToString(usersToCreateSpec))
	klog.V(klogLevel).Infof("Removing groups: %s", entriesToString(groupsToCreateSpec))

	// Delete users / groups.
	fileUsers, _ = removeEntries(fileUsers, usersToCreateSpec)
	fileGroups, _ = removeEntries(fileGroups, groupsToCreateSpec)

	klog.V(klogLevel).Infof("Writing %q and %q", pathUsers, pathGroups)

	// Write the files.
	if err := writeFile(fUsers, fileUsers); err != nil {
		return err
	}
	if err := writeFile(fGroups, fileGroups); err != nil {
		return err
	}

	return nil
}


parseLoginDefs

// parseLoginDefs can be used to parse an /etc/login.defs file and obtain system ranges for UID and GID.
// Passing an empty string will return the defaults. The defaults are 100-999 for both UID and GID.
func parseLoginDefs(file string) (*limits, error) {
	l := *defaultLimits
	if len(file) == 0 {
		return &l, nil
	}
	var mapping = map[string]*int64{
		"SYS_UID_MIN": &l.minUID,
		"SYS_UID_MAX": &l.maxUID,
		"SYS_GID_MIN": &l.minGID,
		"SYS_GID_MAX": &l.maxGID,
	}
	lines := strings.Split(file, "\n")
	for i, line := range lines {
		for k, v := range mapping {
			// A line must start with one of the definitions
			if !strings.HasPrefix(line, k) {
				continue
			}
			line = strings.TrimPrefix(line, k)
			line = strings.TrimSpace(line)
			val, err := strconv.ParseInt(line, 10, 64)
			if err != nil {
				return nil, errors.Wrapf(err, "could not parse value for %s at line %d", k, i)
			}
			*v = val
		}
	}
	return &l, nil
}


parseEntries

// parseEntries can be used to parse an /etc/passwd or /etc/group file as their format is similar.
// It returns a slice of entries obtained from the file.
// https://www.cyberciti.biz/faq/understanding-etcpasswd-file-format/
// https://www.cyberciti.biz/faq/understanding-etcgroup-file/
func parseEntries(file string, totalFields int) ([]*entry, error) {
	if totalFields != totalFieldsUser && totalFields != totalFieldsGroup {
		return nil, errors.Errorf("unsupported total fields for entry parsing: %d", totalFields)
	}
	lines := strings.Split(file, "\n")
	entries := []*entry{}
	for i, line := range lines {
		line = strings.TrimSpace(line)
		if len(line) == 0 {
			continue
		}
		fields := strings.Split(line, ":")
		if len(fields) != totalFields {
			return nil, errors.Errorf("entry must have %d fields separated by ':', "+
				"got %d at line %d: %s", totalFields, len(fields), i, line)
		}
		id, err := strconv.ParseInt(fields[2], 10, 64)
		if err != nil {
			return nil, errors.Wrapf(err, "error parsing id at line %d", i)
		}
		entry := &entry{name: fields[0], id: id}
		if totalFields == totalFieldsGroup {
			entry.userNames = strings.Split(fields[3], ",")
		} else {
			gid, err := strconv.ParseInt(fields[3], 10, 64)
			if err != nil {
				return nil, errors.Wrapf(err, "error parsing GID at line %d", i)
			}
			entry.gid = gid
			entry.shell = fields[6]
		}
		entries = append(entries, entry)
	}
	return entries, nil
}


validateEntries

// validateEntries takes user and group entries and validates if these entries are valid based on limits,
// mapping between users and groups and specs. Returns slices of missing user and group entries that must be created.
// Returns an error if existing users and groups do not match requirements.
func validateEntries(users, groups []*entry, limits *limits) ([]*entry, []*entry, error) {
	u := []*entry{}
	g := []*entry{}
	// Validate users
	for _, uc := range usersToCreateSpec {
		for _, user := range users {
			if uc.name != user.name {
				continue
			}
			// Found existing user
			if user.id < limits.minUID || user.id > limits.maxUID {
				return nil, nil, errors.Errorf("UID %d for user %q is outside the system UID range: %d - %d",
					user.id, user.name, limits.minUID, limits.maxUID)
			}
			if user.shell != noshell {
				return nil, nil, errors.Errorf("user %q has unexpected shell %q; expected %q",
					user.name, user.shell, noshell)
			}
			for _, g := range groups {
				if g.id != user.gid {
					continue
				}
				// Found matching group GID for user GID
				if g.name != uc.name {
					return nil, nil, errors.Errorf("user %q has GID %d but the group with that GID is not named %q",
						uc.name, g.id, uc.name)
				}
				goto skipUser // Valid group GID and name; skip
			}
			return nil, nil, errors.Errorf("could not find group with GID %d for user %q", user.gid, user.name)
		}
		u = append(u, uc)
	skipUser:
	}
	// validate groups
	for _, gc := range groupsToCreateSpec {
		for _, group := range groups {
			if gc.name != group.name {
				continue
			}
			if group.id < limits.minGID || group.id > limits.maxGID {
				return nil, nil, errors.Errorf("GID %d for user %q is outside the system UID range: %d - %d",
					group.id, group.name, limits.minGID, limits.maxGID)
			}
			u1 := strings.Join(gc.userNames, ",")
			u2 := strings.Join(group.userNames, ",")
			if u1 != u2 {
				return nil, nil, errors.Errorf("expected users %q for group %q; got %q",
					u1, gc.name, u2)
			}
			goto skipGroup // group has valid users; skip
		}
		g = append(g, gc)
	skipGroup:
	}
	return u, g, nil
}


allocateIDs

// allocateIDs takes a list of entries and based on minimum and maximum ID allocates a "total" of IDs.
func allocateIDs(entries []*entry, min, max int64, total int) ([]int64, error) {
	if total == 0 {
		return []int64{}, nil
	}
	ids := make([]int64, 0, total)
	for i := min; i < max+1; i++ {
		i64 := int64(i)
		for _, e := range entries {
			if i64 == e.id {
				goto continueLoop
			}
		}
		ids = append(ids, i64)
		if len(ids) == total {
			return ids, nil
		}
	continueLoop:
	}
	return nil, errors.Errorf("could not allocate %d IDs based on existing entries in the range: %d - %d",
		total, min, max)
}


addEntries

// addEntries takes /etc/passwd or /etc/group file content and appends entries to it based
// on a createEntry function. Returns the updated contents of the file.
func addEntries(file string, entries []*entry, createEntry func(*entry) string) string {
	out := file
	newLines := make([]string, 0, len(entries))
	for _, e := range entries {
		newLines = append(newLines, createEntry(e))
	}
	newLinesStr := ""
	if len(newLines) > 0 {
		if !strings.HasSuffix(out, "\n") { // Append a new line if its missing.
			newLinesStr = "\n"
		}
		newLinesStr += strings.Join(newLines, "\n") + "\n"
	}
	return out + newLinesStr
}


removeEntries

// removeEntries takes /etc/passwd or /etc/group file content and deletes entries from them
// by name matching. Returns the updated contents of the file and the number of entries removed.
func removeEntries(file string, entries []*entry) (string, int) {
	lines := strings.Split(file, "\n")
	total := len(lines) - len(entries)
	if total < 0 {
		total = 0
	}
	newLines := make([]string, 0, total)
	removed := 0
	for _, line := range lines {
		for _, entry := range entries {
			if strings.HasPrefix(line, entry.name+":") {
				removed++
				goto continueLoop
			}
		}
		newLines = append(newLines, line)
	continueLoop:
	}
	return strings.Join(newLines, "\n"), removed
}


assignUserAndGroupIDs

// assignUserAndGroupIDs takes the list of existing groups, the users and groups to be created,
// and assigns UIDs and GIDs to the users and groups to be created based on a list of provided UIDs and GIDs.
// Returns an error if not enough UIDs or GIDs are passed. It does not perform any other validation.
func assignUserAndGroupIDs(groups, usersToCreate, groupsToCreate []*entry, uids, gids []int64) error {
	if len(gids) < len(groupsToCreate) {
		return errors.Errorf("not enough GIDs to assign to groups: have %d, want %d", len(gids), len(groupsToCreate))
	}
	if len(uids) < len(usersToCreate) {
		return errors.Errorf("not enough UIDs to assign to users: have %d, want %d", len(uids), len(usersToCreate))
	}
	for i := range groupsToCreate {
		groupsToCreate[i].id = gids[i]
	}
	// Concat the list of old and new groups to find a matching GID.
	groupsConcat := append([]*entry{}, groups...)
	groupsConcat = append(groupsConcat, groupsToCreate...)
	for i := range usersToCreate {
		usersToCreate[i].id = uids[i]
		for _, g := range groupsConcat {
			if usersToCreate[i].name == g.name {
				usersToCreate[i].gid = g.id
				break
			}
		}
	}
	return nil
}


createGroup

// createGroup is a helper function to produce a group from entry.
func createGroup(e *entry) string {
	return fmt.Sprintf("%s:x:%d:%s", e.name, e.id, strings.Join(e.userNames, ","))
}


createUser

// createUser is a helper function to produce a user from entry.
func createUser(e *entry) string {
	return fmt.Sprintf("%s:x:%d:%d:::/bin/false", e.name, e.id, e.gid)
}


entriesToEntryMap

// entriesToEntryMap takes a list of entries and prepares an EntryMap object.
func entriesToEntryMap(entries, spec []*entry) (*EntryMap, error) {
	m := map[string]*entry{}
	for _, spec := range spec {
		for _, e := range entries {
			if spec.name == e.name {
				entry := *e
				m[e.name] = &entry
				goto continueLoop
			}
		}
		return nil, errors.Errorf("could not find entry %q in the list", spec.name)
	continueLoop:
	}
	return &EntryMap{entries: m}, nil
}


entriesToString

// entriesToString is a utility to convert a list of entries to string.
func entriesToString(entries []*entry) string {
	lines := make([]string, 0, len(entries))
	for _, e := range entries {
		lines = append(lines, e.name)
	}
	sort.Strings(lines)
	return strings.Join(lines, ",")
}


openFileWithLock

// openFileWithLock opens the file at path by acquiring an exclive write lock.
// The returned close() function should be called to release the lock and close the file.
// If a lock cannot be obtained the function fails after a period of time.
func openFileWithLock(path string) (f *os.File, close func(), err error) {
	f, err = os.OpenFile(path, os.O_RDWR, os.ModePerm)
	if err != nil {
		return nil, nil, err
	}
	deadline := time.Now().Add(time.Second * 5)
	for {
		// If another process is holding a write lock, this call will exit
		// with an error. F_SETLK is used instead of F_SETLKW to avoid
		// the case where a runaway process grabs the exclusive lock and
		// blocks this call indefinitely.
		// https://man7.org/linux/man-pages/man2/fcntl.2.html
		lock := syscall.Flock_t{Type: syscall.F_WRLCK}
		if err = syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &lock); err == nil {
			break
		}
		time.Sleep(200 * time.Millisecond)
		if time.Now().After(deadline) {
			err = errors.Wrapf(err, "timeout attempting to obtain lock on file %q", path)
			break
		}
	}
	if err != nil {
		f.Close()
		return nil, nil, err
	}
	close = func() {
		// This function should be called once operations with the file are finished.
		// It unlocks the file and closes it.
		unlock := syscall.Flock_t{Type: syscall.F_UNLCK}
		syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &unlock)
		f.Close()
	}
	return f, close, nil
}


readFile

// readFile reads a File into a string.
func readFile(f *os.File) (string, error) {
	buf := bytes.NewBuffer(nil)
	if _, err := f.Seek(0, io.SeekStart); err != nil {
		return "", err
	}
	if _, err := io.Copy(buf, f); err != nil {
		return "", err
	}
	return buf.String(), nil
}


writeFile

// writeFile writes a string to a File.
func writeFile(f *os.File, str string) error {
	if _, err := f.Seek(0, io.SeekStart); err != nil {
		return err
	}
	if _, err := f.Write([]byte(str)); err != nil {
		return err
	}
	if err := f.Truncate(int64(len(str))); err != nil {
		return err
	}
	return nil
}


UpdatePathOwnerAndPermissions

// UpdatePathOwnerAndPermissions updates the owner and permissions of the given path.
// If the path is a directory it is not recursively updated.
func UpdatePathOwnerAndPermissions(path string, uid, gid int64, perms uint32) error {
	if err := os.Chown(path, int(uid), int(gid)); err != nil {
		return errors.Wrapf(err, "failed to update owner of %q to uid: %d and gid: %d", path, uid, gid)
	}
	fm := os.FileMode(perms)
	if err := os.Chmod(path, fm); err != nil {
		return errors.Wrapf(err, "failed to update permissions of %q to %s", path, fm.String())
	}
	return nil
}


UpdatePathOwner

// UpdatePathOwner recursively updates the owners of a directory.
// It is equivalent to calling `chown -R uid:gid /path/to/dir`.
func UpdatePathOwner(dirPath string, uid, gid int64) error {
	err := filepath.WalkDir(dirPath, func(path string, d os.DirEntry, err error) error {
		if err := os.Chown(path, int(uid), int(gid)); err != nil {
			return errors.Wrapf(err, "failed to update owner of %q to uid: %d and gid: %d", path, uid, gid)
		}
		return nil
	})
	return err
}


ID

// ID is a NO-OP on non-Linux.
func (*EntryMap) ID(string) *int64 {
	return nil
}


String

// String is NO-OP on non-Linux.
func (*EntryMap) String() string {
	return ""
}


AddUsersAndGroups

// AddUsersAndGroups is a NO-OP on non-Linux.
func AddUsersAndGroups() (*UsersAndGroups, error) {
	return nil, nil
}


RemoveUsersAndGroups

// RemoveUsersAndGroups is a NO-OP on non-Linux.
func RemoveUsersAndGroups() error {
	return nil
}


UpdatePathOwnerAndPermissions

// UpdatePathOwnerAndPermissions is a NO-OP on non-Linux.
func UpdatePathOwnerAndPermissions(path string, uid, gid int64, perms uint32) error {
	return nil
}


UpdatePathOwner

// UpdatePathOwner is a NO-OP on non-Linux.
func UpdatePathOwner(dirPath string, uid, gid int64) error {
	return nil
}


KubernetesReleaseVersion

// KubernetesReleaseVersion is helper function that can fetch
// available version information from release servers based on
// label names, like "stable" or "latest".
//
// If argument is already semantic version string, it
// will return same string.
//
// In case of labels, it tries to fetch from release
// servers and then return actual semantic version.
//
// Available names on release servers:
//
//	stable      (latest stable release)
//	stable-1    (latest stable release in 1.x)
//	stable-1.0  (and similarly 1.1, 1.2, 1.3, ...)
//	latest      (latest release, including alpha/beta)
//	latest-1    (latest release in 1.x, including alpha/beta)
//	latest-1.0  (and similarly 1.1, 1.2, 1.3, ...)
func KubernetesReleaseVersion(version string) (string, error) {
	return kubernetesReleaseVersion(version, fetchFromURL)
}


kubernetesReleaseVersion

// kubernetesReleaseVersion is a helper function to fetch
// available version information. Used for testing to eliminate
// the need for internet calls.
func kubernetesReleaseVersion(version string, fetcher func(string, time.Duration) (string, error)) (string, error) {
	ver := normalizedBuildVersion(version)
	if len(ver) != 0 {
		return ver, nil
	}

	bucketURL, versionLabel, err := splitVersion(version)
	if err != nil {
		return "", err
	}

	// revalidate, if exact build from e.g. CI bucket requested.
	ver = normalizedBuildVersion(versionLabel)
	if len(ver) != 0 {
		return ver, nil
	}

	// kubeReleaseLabelRegex matches labels such as: latest, latest-1, latest-1.10
	if kubeReleaseLabelRegex.MatchString(versionLabel) {
		// Try to obtain a client version.
		// pkgversion.Get().String() should always return a correct version added by the golang
		// linker and the build system. The version can still be missing when doing unit tests
		// on individual packages.
		clientVersion, clientVersionErr := kubeadmVersion(pkgversion.Get().String())
		// Fetch version from the internet.
		url := fmt.Sprintf("%s/%s.txt", bucketURL, versionLabel)
		body, err := fetcher(url, getReleaseVersionTimeout)
		if err != nil {
			if clientVersionErr == nil {
				// Handle air-gapped environments by falling back to the client version.
				klog.Warningf("could not fetch a Kubernetes version from the internet: %v", err)
				klog.Warningf("falling back to the local client version: %s", clientVersion)
				return kubernetesReleaseVersion(clientVersion, fetcher)
			}
		}

		if clientVersionErr != nil {
			if err != nil {
				klog.Warningf("could not obtain neither client nor remote version; fall back to: %s", constants.CurrentKubernetesVersion)
				return kubernetesReleaseVersion(constants.CurrentKubernetesVersion.String(), fetcher)
			}

			klog.Warningf("could not obtain client version; using remote version: %s", body)
			return kubernetesReleaseVersion(body, fetcher)
		}

		// both the client and the remote version are obtained; validate them and pick a stable version
		body, err = validateStableVersion(body, clientVersion)
		if err != nil {
			return "", err
		}
		// Re-validate received version and return.
		return kubernetesReleaseVersion(body, fetcher)
	}
	return "", errors.Errorf("version %q doesn't match patterns for neither semantic version nor labels (stable, latest, ...)", version)
}


KubernetesVersionToImageTag

// KubernetesVersionToImageTag is helper function that replaces all
// non-allowed symbols in tag strings with underscores.
// Image tag can only contain lowercase and uppercase letters, digits,
// underscores, periods and dashes.
// Current usage is for CI images where all of symbols except '+' are valid,
// but function is for generic usage where input can't be always pre-validated.
func KubernetesVersionToImageTag(version string) string {
	allowed := regexp.MustCompile(`[^-\w.]`)
	return allowed.ReplaceAllString(version, "_")
}


KubernetesIsCIVersion

// KubernetesIsCIVersion checks if user requested CI version
func KubernetesIsCIVersion(version string) bool {
	subs := kubeBucketPrefixes.FindAllStringSubmatch(version, 1)
	if len(subs) == 1 && len(subs[0]) == 4 && strings.HasPrefix(subs[0][2], "ci") {
		return true
	}
	return false
}


normalizedBuildVersion

// Internal helper: returns normalized build version (with "v" prefix if needed)
// If input doesn't match known version pattern, returns empty string.
func normalizedBuildVersion(version string) string {
	if kubeReleaseRegex.MatchString(version) {
		if strings.HasPrefix(version, "v") {
			return version
		}
		return "v" + version
	}
	return ""
}


splitVersion

// Internal helper: split version parts,
// Return base URL and cleaned-up version
func splitVersion(version string) (string, string, error) {
	var bucketURL, urlSuffix string
	subs := kubeBucketPrefixes.FindAllStringSubmatch(version, 1)
	if len(subs) != 1 || len(subs[0]) != 4 {
		return "", "", errors.Errorf("invalid version %q", version)
	}

	switch {
	case strings.HasPrefix(subs[0][2], "ci"):
		// Just use whichever the user specified
		urlSuffix = subs[0][2]
		bucketURL = kubeCIBucketURL
	default:
		urlSuffix = "release"
		bucketURL = kubeReleaseBucketURL
	}
	url := fmt.Sprintf("%s/%s", bucketURL, urlSuffix)
	return url, subs[0][3], nil
}


fetchFromURL

// Internal helper: return content of URL
func fetchFromURL(url string, timeout time.Duration) (string, error) {
	klog.V(2).Infof("fetching Kubernetes version from URL: %s", url)
	client := &http.Client{Timeout: timeout, Transport: netutil.SetOldTransportDefaults(&http.Transport{})}
	resp, err := client.Get(url)
	if err != nil {
		return "", errors.Errorf("unable to get URL %q: %s", url, err.Error())
	}
	defer resp.Body.Close()
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return "", errors.Errorf("unable to read content of URL %q: %s", url, err.Error())
	}
	bodyString := strings.TrimSpace(string(body))

	if resp.StatusCode != http.StatusOK {
		msg := fmt.Sprintf("unable to fetch file. URL: %q, status: %v", url, resp.Status)
		return bodyString, errors.New(msg)
	}
	return bodyString, nil
}


kubeadmVersion

// kubeadmVersion returns the version of the client without metadata.
func kubeadmVersion(info string) (string, error) {
	v, err := versionutil.ParseSemantic(info)
	if err != nil {
		return "", errors.Wrap(err, "kubeadm version error")
	}
	// There is no utility in versionutil to get the version without the metadata,
	// so this needs some manual formatting.
	// Discard offsets after a release label and keep the labels down to e.g. `alpha.0` instead of
	// including the offset e.g. `alpha.0.206`. This is done to comply with GCR image tags.
	pre := v.PreRelease()
	patch := v.Patch()
	if len(pre) > 0 {
		if patch > 0 {
			// If the patch version is more than zero, decrement it and remove the label.
			// this is done to comply with the latest stable patch release.
			patch = patch - 1
			pre = ""
		} else {
			split := strings.Split(pre, ".")
			if len(split) > 2 {
				pre = split[0] + "." + split[1] // Exclude the third element
			} else if len(split) < 2 {
				pre = split[0] + ".0" // Append .0 to a partial label
			}
			pre = "-" + pre
		}
	}
	vStr := fmt.Sprintf("v%d.%d.%d%s", v.Major(), v.Minor(), patch, pre)
	return vStr, nil
}


validateStableVersion

// Validate if the remote version is one Minor release newer than the client version.
// This is done to conform with "stable-X" and only allow remote versions from
// the same Patch level release.
func validateStableVersion(remoteVersion, clientVersion string) (string, error) {
	verRemote, err := versionutil.ParseGeneric(remoteVersion)
	if err != nil {
		return "", errors.Wrap(err, "remote version error")
	}
	verClient, err := versionutil.ParseGeneric(clientVersion)
	if err != nil {
		return "", errors.Wrap(err, "client version error")
	}
	// If the remote Major version is bigger or if the Major versions are the same,
	// but the remote Minor is bigger use the client version release. This handles Major bumps too.
	if verClient.Major() < verRemote.Major() ||
		(verClient.Major() == verRemote.Major()) && verClient.Minor() < verRemote.Minor() {
		estimatedRelease := fmt.Sprintf("stable-%d.%d", verClient.Major(), verClient.Minor())
		klog.Infof("remote version is much newer: %s; falling back to: %s", remoteVersion, estimatedRelease)
		return estimatedRelease, nil
	}
	return remoteVersion, nil
}


main

func main() {
	kubeadmutil.CheckErr(app.Run())
}


func main() {
	flags := pflag.NewFlagSet("kubectl-convert", pflag.ExitOnError)
	pflag.CommandLine = flags

	kubeConfigFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag()
	kubeConfigFlags.AddFlags(flags)
	matchVersionKubeConfigFlags := cmdutil.NewMatchVersionFlags(kubeConfigFlags)

	f := cmdutil.NewFactory(matchVersionKubeConfigFlags)
	cmd := convert.NewCmdConvert(f, genericclioptions.IOStreams{In: os.Stdin, Out: os.Stdout, ErrOut: os.Stderr})
	matchVersionKubeConfigFlags.AddFlags(cmd.PersistentFlags())
	code := cli.Run(cmd)
	os.Exit(code)
}


func main() {
	command := cmd.NewDefaultKubectlCommand()
	if err := cli.RunNoErrOutput(command); err != nil {
		// Pretty-print the error and exit with an error.
		util.CheckErr(err)
	}
}


BuildAuth

// BuildAuth creates an authenticator, an authorizer, and a matching authorizer attributes getter compatible with the kubelet's needs
// It returns AuthInterface, a run method to start internal controllers (like cert reloading) and error.
func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubeletconfig.KubeletConfiguration) (server.AuthInterface, func(<-chan struct{}), error) {
	// Get clients, if provided
	var (
		tokenClient authenticationclient.AuthenticationV1Interface
		sarClient   authorizationclient.AuthorizationV1Interface
	)
	if client != nil && !reflect.ValueOf(client).IsNil() {
		tokenClient = client.AuthenticationV1()
		sarClient = client.AuthorizationV1()
	}

	authenticator, runAuthenticatorCAReload, err := BuildAuthn(tokenClient, config.Authentication)
	if err != nil {
		return nil, nil, err
	}

	attributes := server.NewNodeAuthorizerAttributesGetter(nodeName)

	authorizer, err := BuildAuthz(sarClient, config.Authorization)
	if err != nil {
		return nil, nil, err
	}

	return server.NewKubeletAuth(authenticator, attributes, authorizer), runAuthenticatorCAReload, nil
}


BuildAuthn

// BuildAuthn creates an authenticator compatible with the kubelet's needs
func BuildAuthn(client authenticationclient.AuthenticationV1Interface, authn kubeletconfig.KubeletAuthentication) (authenticator.Request, func(<-chan struct{}), error) {
	var dynamicCAContentFromFile *dynamiccertificates.DynamicFileCAContent
	var err error
	if len(authn.X509.ClientCAFile) > 0 {
		dynamicCAContentFromFile, err = dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", authn.X509.ClientCAFile)
		if err != nil {
			return nil, nil, err
		}
	}

	authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{
		Anonymous:                          authn.Anonymous.Enabled,
		CacheTTL:                           authn.Webhook.CacheTTL.Duration,
		ClientCertificateCAContentProvider: dynamicCAContentFromFile,
	}

	if authn.Webhook.Enabled {
		if client == nil {
			return nil, nil, errors.New("no client provided, cannot use webhook authentication")
		}
		authenticatorConfig.WebhookRetryBackoff = genericoptions.DefaultAuthWebhookRetryBackoff()
		authenticatorConfig.TokenAccessReviewClient = client
	}

	authenticator, _, err := authenticatorConfig.New()
	if err != nil {
		return nil, nil, err
	}

	return authenticator, func(stopCh <-chan struct{}) {
		// generate a context from stopCh. This is to avoid modifying files which are relying on this method
		// TODO: See if we can pass ctx to the current method
		ctx, cancel := context.WithCancel(context.Background())
		go func() {
			select {
			case <-stopCh:
				cancel() // stopCh closed, so cancel our context
			case <-ctx.Done():
			}
		}()
		if dynamicCAContentFromFile != nil {
			go dynamicCAContentFromFile.Run(ctx, 1)
		}
	}, err
}


BuildAuthz

// BuildAuthz creates an authorizer compatible with the kubelet's needs
func BuildAuthz(client authorizationclient.AuthorizationV1Interface, authz kubeletconfig.KubeletAuthorization) (authorizer.Authorizer, error) {
	switch authz.Mode {
	case kubeletconfig.KubeletAuthorizationModeAlwaysAllow:
		return authorizerfactory.NewAlwaysAllowAuthorizer(), nil

	case kubeletconfig.KubeletAuthorizationModeWebhook:
		if client == nil {
			return nil, errors.New("no client provided, cannot use webhook authorization")
		}
		authorizerConfig := authorizerfactory.DelegatingAuthorizerConfig{
			SubjectAccessReviewClient: client,
			AllowCacheTTL:             authz.Webhook.CacheAuthorizedTTL.Duration,
			DenyCacheTTL:              authz.Webhook.CacheUnauthorizedTTL.Duration,
			WebhookRetryBackoff:       genericoptions.DefaultAuthWebhookRetryBackoff(),
		}
		return authorizerConfig.New()

	case "":
		return nil, fmt.Errorf("no authorization mode specified")

	default:
		return nil, fmt.Errorf("unknown authorization mode %s", authz.Mode)

	}
}


initForOS

func initForOS(service bool, priorityClass string) error {
	return nil
}


getPriorityValue

// getPriorityValue returns the value associated with a Windows process priorityClass
// Ref: https://docs.microsoft.com/en-us/windows/win32/cimwin32prov/setpriority-method-in-class-win32-process
func getPriorityValue(priorityClassName string) uint32 {
	var priorityClassMap = map[string]uint32{
		"IDLE_PRIORITY_CLASS":         uint32(windows.IDLE_PRIORITY_CLASS),
		"BELOW_NORMAL_PRIORITY_CLASS": uint32(windows.BELOW_NORMAL_PRIORITY_CLASS),
		"NORMAL_PRIORITY_CLASS":       uint32(windows.NORMAL_PRIORITY_CLASS),
		"ABOVE_NORMAL_PRIORITY_CLASS": uint32(windows.ABOVE_NORMAL_PRIORITY_CLASS),
		"HIGH_PRIORITY_CLASS":         uint32(windows.HIGH_PRIORITY_CLASS),
		"REALTIME_PRIORITY_CLASS":     uint32(windows.REALTIME_PRIORITY_CLASS),
	}
	return priorityClassMap[priorityClassName]
}


createWindowsJobObject

// createWindowsJobObject creates a new Job Object
// (https://docs.microsoft.com/en-us/windows/win32/procthread/job-objects),
// and specifies the priority class for the job object to the specified value.
// A job object is used here so that any spawned processes such as powershell or
// wmic are created at the specified thread priority class.
// Running kubelet with above normal / high priority  can help improve
// responsiveness on machines with high CPU utilization.
func createWindowsJobObject(pc uint32) (windows.Handle, error) {
	job, err := windows.CreateJobObject(nil, nil)
	if err != nil {
		return windows.InvalidHandle, fmt.Errorf("windows.CreateJobObject failed: %w", err)
	}
	limitInfo := windows.JOBOBJECT_BASIC_LIMIT_INFORMATION{
		LimitFlags:    windows.JOB_OBJECT_LIMIT_PRIORITY_CLASS,
		PriorityClass: pc,
	}
	if _, err := windows.SetInformationJobObject(
		job,
		windows.JobObjectBasicLimitInformation,
		uintptr(unsafe.Pointer(&limitInfo)),
		uint32(unsafe.Sizeof(limitInfo))); err != nil {
		return windows.InvalidHandle, fmt.Errorf("windows.SetInformationJobObject failed: %w", err)
	}
	return job, nil
}


initForOS

func initForOS(windowsService bool, windowsPriorityClass string) error {
	priority := getPriorityValue(windowsPriorityClass)
	if priority == 0 {
		return fmt.Errorf("unknown priority class %s, valid ones are available at "+
			"https://docs.microsoft.com/en-us/windows/win32/procthread/scheduling-priorities", windowsPriorityClass)
	}
	klog.InfoS("Creating a Windows job object and adding kubelet process to it", "windowsPriorityClass", windowsPriorityClass)
	job, err := createWindowsJobObject(priority)
	if err != nil {
		return err
	}
	if err := windows.AssignProcessToJobObject(job, windows.CurrentProcess()); err != nil {
		return fmt.Errorf("windows.AssignProcessToJobObject failed: %w", err)
	}

	if windowsService {
		return service.InitService(serviceName)
	}
	return nil
}


NewContainerRuntimeOptions

// NewContainerRuntimeOptions will create a new ContainerRuntimeOptions with
// default values.
func NewContainerRuntimeOptions() *config.ContainerRuntimeOptions {
	return &config.ContainerRuntimeOptions{
		PodSandboxImage: defaultPodSandboxImage,
	}
}


AddGlobalFlags

// AddGlobalFlags explicitly registers flags that libraries (glog, verflag, etc.) register
// against the global flagsets from "flag" and "github.com/spf13/pflag".
// We do this in order to prevent unwanted flags from leaking into the Kubelet's flagset.
func AddGlobalFlags(fs *pflag.FlagSet) {
	addCadvisorFlags(fs)
	addCredentialProviderFlags(fs)
	verflag.AddFlags(fs)
	logs.AddFlags(fs, logs.SkipLoggingConfigurationFlags())
}


normalize

// normalize replaces underscores with hyphens
// we should always use hyphens instead of underscores when registering kubelet flags
func normalize(s string) string {
	return strings.Replace(s, "_", "-", -1)
}


register

// register adds a flag to local that targets the Value associated with the Flag named globalName in global
func register(global *flag.FlagSet, local *pflag.FlagSet, globalName string) {
	if f := global.Lookup(globalName); f != nil {
		pflagFlag := pflag.PFlagFromGoFlag(f)
		pflagFlag.Name = normalize(pflagFlag.Name)
		local.AddFlag(pflagFlag)
	} else {
		panic(fmt.Sprintf("failed to find flag in global flagset (flag): %s", globalName))
	}
}


pflagRegister

// pflagRegister adds a flag to local that targets the Value associated with the Flag named globalName in global
func pflagRegister(global, local *pflag.FlagSet, globalName string) {
	if f := global.Lookup(globalName); f != nil {
		f.Name = normalize(f.Name)
		local.AddFlag(f)
	} else {
		panic(fmt.Sprintf("failed to find flag in global flagset (pflag): %s", globalName))
	}
}


registerDeprecated

// registerDeprecated registers the flag with register, and then marks it deprecated
func registerDeprecated(global *flag.FlagSet, local *pflag.FlagSet, globalName, deprecated string) {
	register(global, local, globalName)
	local.Lookup(normalize(globalName)).Deprecated = deprecated
}


addCredentialProviderFlags

// addCredentialProviderFlags adds flags from k8s.io/kubernetes/pkg/credentialprovider
func addCredentialProviderFlags(fs *pflag.FlagSet) {
	// lookup flags in global flag set and re-register the values with our flagset
	global := pflag.CommandLine
	local := pflag.NewFlagSet(os.Args[0], pflag.ExitOnError)

	addLegacyCloudProviderCredentialProviderFlags(global, local)

	fs.AddFlagSet(local)
}


addCadvisorFlags

// addCadvisorFlags adds flags from cadvisor
func addCadvisorFlags(fs *pflag.FlagSet) {
	// lookup flags in global flag set and re-register the values with our flagset
	global := flag.CommandLine
	local := pflag.NewFlagSet(os.Args[0], pflag.ExitOnError)

	// e2e node tests rely on this
	register(global, local, "housekeeping_interval")

	// These flags were implicit from cadvisor, and are mistakes that should be registered deprecated:
	const deprecated = "This is a cadvisor flag that was mistakenly registered with the Kubelet. Due to legacy concerns, it will follow the standard CLI deprecation timeline before being removed."

	registerDeprecated(global, local, "application_metrics_count_limit", deprecated)
	registerDeprecated(global, local, "boot_id_file", deprecated)
	registerDeprecated(global, local, "container_hints", deprecated)
	registerDeprecated(global, local, "containerd", deprecated)
	registerDeprecated(global, local, "enable_load_reader", deprecated)
	registerDeprecated(global, local, "event_storage_age_limit", deprecated)
	registerDeprecated(global, local, "event_storage_event_limit", deprecated)
	registerDeprecated(global, local, "global_housekeeping_interval", deprecated)
	registerDeprecated(global, local, "log_cadvisor_usage", deprecated)
	registerDeprecated(global, local, "machine_id_file", deprecated)
	registerDeprecated(global, local, "storage_driver_user", deprecated)
	registerDeprecated(global, local, "storage_driver_password", deprecated)
	registerDeprecated(global, local, "storage_driver_host", deprecated)
	registerDeprecated(global, local, "storage_driver_db", deprecated)
	registerDeprecated(global, local, "storage_driver_table", deprecated)
	registerDeprecated(global, local, "storage_driver_secure", deprecated)
	registerDeprecated(global, local, "storage_driver_buffer_duration", deprecated)
	registerDeprecated(global, local, "containerd-namespace", deprecated)

	// finally, add cadvisor flags to the provided flagset
	fs.AddFlagSet(local)
}


func addCadvisorFlags(fs *pflag.FlagSet) {
}


func addLegacyCloudProviderCredentialProviderFlags(global, local *pflag.FlagSet) {
	// no-op when no legacy providers are compiled in
}


func addLegacyCloudProviderCredentialProviderFlags(global, local *pflag.FlagSet) {
	// TODO(#58034): This is not a static file, so it's not quite as straightforward as --google-json-key.
	// We need to figure out how ACR users can dynamically provide pull credentials before we can deprecate this.
	pflagRegister(global, local, "azure-container-registry-config")
}


NewKubeletFlags

// NewKubeletFlags will create a new KubeletFlags with default values
func NewKubeletFlags() *KubeletFlags {
	return &KubeletFlags{
		ContainerRuntimeOptions: *NewContainerRuntimeOptions(),
		CertDirectory:           "/var/lib/kubelet/pki",
		RootDirectory:           filepath.Clean(defaultRootDir),
		MaxContainerCount:       -1,
		MaxPerPodContainerCount: 1,
		MinimumGCAge:            metav1.Duration{Duration: 0},
		RegisterSchedulable:     true,
		NodeLabels:              make(map[string]string),
	}
}


ValidateKubeletFlags

// ValidateKubeletFlags validates Kubelet's configuration flags and returns an error if they are invalid.
func ValidateKubeletFlags(f *KubeletFlags) error {
	unknownLabels := sets.NewString()
	invalidLabelErrs := make(map[string][]string)
	for k, v := range f.NodeLabels {
		if isKubernetesLabel(k) && !kubeletapis.IsKubeletLabel(k) {
			unknownLabels.Insert(k)
		}

		if errs := validation.IsQualifiedName(k); len(errs) > 0 {
			invalidLabelErrs[k] = append(invalidLabelErrs[k], errs...)
		}
		if errs := validation.IsValidLabelValue(v); len(errs) > 0 {
			invalidLabelErrs[v] = append(invalidLabelErrs[v], errs...)
		}
	}
	if len(unknownLabels) > 0 {
		return fmt.Errorf("unknown 'kubernetes.io' or 'k8s.io' labels specified with --node-labels: %v\n--node-labels in the 'kubernetes.io' namespace must begin with an allowed prefix (%s) or be in the specifically allowed set (%s)", unknownLabels.List(), strings.Join(kubeletapis.KubeletLabelNamespaces(), ", "), strings.Join(kubeletapis.KubeletLabels(), ", "))
	}
	if len(invalidLabelErrs) > 0 {
		labelErrs := []string{}
		for k, v := range invalidLabelErrs {
			labelErrs = append(labelErrs, fmt.Sprintf("'%s' - %s", k, strings.Join(v, ", ")))
		}
		return fmt.Errorf("invalid node labels: %s", strings.Join(labelErrs, "; "))
	}
	return nil
}


isKubernetesLabel

func isKubernetesLabel(key string) bool {
	namespace := getLabelNamespace(key)
	if namespace == "kubernetes.io" || strings.HasSuffix(namespace, ".kubernetes.io") {
		return true
	}
	if namespace == "k8s.io" || strings.HasSuffix(namespace, ".k8s.io") {
		return true
	}
	return false
}


getLabelNamespace

func getLabelNamespace(key string) string {
	if parts := strings.SplitN(key, "/", 2); len(parts) == 2 {
		return parts[0]
	}
	return ""
}


NewKubeletConfiguration

// NewKubeletConfiguration will create a new KubeletConfiguration with default values
func NewKubeletConfiguration() (*kubeletconfig.KubeletConfiguration, error) {
	scheme, _, err := kubeletscheme.NewSchemeAndCodecs()
	if err != nil {
		return nil, err
	}
	versioned := &v1beta1.KubeletConfiguration{}
	scheme.Default(versioned)
	config := &kubeletconfig.KubeletConfiguration{}
	if err := scheme.Convert(versioned, config, nil); err != nil {
		return nil, err
	}
	applyLegacyDefaults(config)
	return config, nil
}


applyLegacyDefaults

// applyLegacyDefaults applies legacy default values to the KubeletConfiguration in order to
// preserve the command line API. This is used to construct the baseline default KubeletConfiguration
// before the first round of flag parsing.
func applyLegacyDefaults(kc *kubeletconfig.KubeletConfiguration) {
	// --anonymous-auth
	kc.Authentication.Anonymous.Enabled = true
	// --authentication-token-webhook
	kc.Authentication.Webhook.Enabled = false
	// --authorization-mode
	kc.Authorization.Mode = kubeletconfig.KubeletAuthorizationModeAlwaysAllow
	// --read-only-port
	kc.ReadOnlyPort = ports.KubeletReadOnlyPort
}


NewKubeletServer

// NewKubeletServer will create a new KubeletServer with default values.
func NewKubeletServer() (*KubeletServer, error) {
	config, err := NewKubeletConfiguration()
	if err != nil {
		return nil, err
	}
	return &KubeletServer{
		KubeletFlags:         *NewKubeletFlags(),
		KubeletConfiguration: *config,
	}, nil
}


ValidateKubeletServer

// ValidateKubeletServer validates configuration of KubeletServer and returns an error if the input configuration is invalid.
func ValidateKubeletServer(s *KubeletServer) error {
	// please add any KubeletConfiguration validation to the kubeletconfigvalidation.ValidateKubeletConfiguration function
	if err := kubeletconfigvalidation.ValidateKubeletConfiguration(&s.KubeletConfiguration, utilfeature.DefaultFeatureGate); err != nil {
		return err
	}
	if err := ValidateKubeletFlags(&s.KubeletFlags); err != nil {
		return err
	}
	return nil
}


AddFlags

// AddFlags adds flags for a specific KubeletServer to the specified FlagSet
func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
	s.KubeletFlags.AddFlags(fs)
	AddKubeletConfigFlags(fs, &s.KubeletConfiguration)
}