isRequired

// Checks whether app armor is required for pod to be run.
func isRequired(pod *v1.Pod) bool {
	for key, value := range pod.Annotations {
		if strings.HasPrefix(key, v1.AppArmorBetaContainerAnnotationKeyPrefix) {
			return value != v1.AppArmorBetaProfileNameUnconfined
		}
	}
	return false
}


GetProfileName

// GetProfileName returns the name of the profile to use with the container.
func GetProfileName(pod *v1.Pod, containerName string) string {
	return GetProfileNameFromPodAnnotations(pod.Annotations, containerName)
}


GetProfileNameFromPodAnnotations

// GetProfileNameFromPodAnnotations gets the name of the profile to use with container from
// pod annotations
func GetProfileNameFromPodAnnotations(annotations map[string]string, containerName string) string {
	return annotations[v1.AppArmorBetaContainerAnnotationKeyPrefix+containerName]
}


NewValidator

// NewValidator is in order to find AppArmor FS
func NewValidator() Validator {
	if err := validateHost(); err != nil {
		return &validator{validateHostErr: err}
	}
	return &validator{}
}


Validate

func (v *validator) Validate(pod *v1.Pod) error {
	if !isRequired(pod) {
		return nil
	}

	if v.ValidateHost() != nil {
		return v.validateHostErr
	}

	var retErr error
	podutil.VisitContainers(&pod.Spec, podutil.AllContainers, func(container *v1.Container, containerType podutil.ContainerType) bool {
		profile := GetProfileName(pod, container.Name)
		retErr = validation.ValidateAppArmorProfileFormat(profile)
		if retErr != nil {
			return false
		}
		// TODO(#64841): This would ideally be part of validation.ValidateAppArmorProfileFormat, but
		// that is called for API validation, and this is tightening validation.
		if strings.HasPrefix(profile, v1.AppArmorBetaProfileNamePrefix) {
			if strings.TrimSpace(strings.TrimPrefix(profile, v1.AppArmorBetaProfileNamePrefix)) == "" {
				retErr = fmt.Errorf("invalid empty AppArmor profile name: %q", profile)
				return false
			}
		}
		return true
	})

	return retErr
}


ValidateHost

// ValidateHost verifies that the host and runtime is capable of enforcing AppArmor profiles.
// Note, this is intentionally only check the host at kubelet startup and never re-evaluates the host
// as the expectation is that the kubelet restart will be needed to enable or disable AppArmor support.
func (v *validator) ValidateHost() error {
	return v.validateHostErr
}


validateHost

// validateHost verifies that the host and runtime is capable of enforcing AppArmor profiles.
func validateHost() error {
	// Check feature-gates
	if !utilfeature.DefaultFeatureGate.Enabled(features.AppArmor) {
		return errors.New("AppArmor disabled by feature-gate")
	}

	// Check build support.
	if isDisabledBuild {
		return errors.New("binary not compiled for linux")
	}

	// Check kernel support.
	if !apparmor.IsEnabled() {
		return errors.New("AppArmor is not enabled on the host")
	}

	return nil
}


init

func init() {
	// If Kubernetes was not built for linux, apparmor is always disabled.
	isDisabledBuild = true
}


NewPodSecurityContextAccessor

// NewPodSecurityContextAccessor returns an accessor for the given pod security context.
// May be initialized with a nil PodSecurityContext.
func NewPodSecurityContextAccessor(podSC *api.PodSecurityContext) PodSecurityContextAccessor {
	return &podSecurityContextWrapper{podSC: podSC}
}


NewPodSecurityContextMutator

// NewPodSecurityContextMutator returns a mutator for the given pod security context.
// May be initialized with a nil PodSecurityContext.
func NewPodSecurityContextMutator(podSC *api.PodSecurityContext) PodSecurityContextMutator {
	return &podSecurityContextWrapper{podSC: podSC}
}


PodSecurityContext

func (w *podSecurityContextWrapper) PodSecurityContext() *api.PodSecurityContext {
	return w.podSC
}


ensurePodSC

func (w *podSecurityContextWrapper) ensurePodSC() {
	if w.podSC == nil {
		w.podSC = &api.PodSecurityContext{}
	}
}


HostNetwork

func (w *podSecurityContextWrapper) HostNetwork() bool {
	if w.podSC == nil {
		return false
	}
	return w.podSC.HostNetwork
}


SetHostNetwork

func (w *podSecurityContextWrapper) SetHostNetwork(v bool) {
	if w.podSC == nil && v == false {
		return
	}
	w.ensurePodSC()
	w.podSC.HostNetwork = v
}


HostPID

func (w *podSecurityContextWrapper) HostPID() bool {
	if w.podSC == nil {
		return false
	}
	return w.podSC.HostPID
}


SetHostPID

func (w *podSecurityContextWrapper) SetHostPID(v bool) {
	if w.podSC == nil && v == false {
		return
	}
	w.ensurePodSC()
	w.podSC.HostPID = v
}


HostIPC

func (w *podSecurityContextWrapper) HostIPC() bool {
	if w.podSC == nil {
		return false
	}
	return w.podSC.HostIPC
}


SetHostIPC

func (w *podSecurityContextWrapper) SetHostIPC(v bool) {
	if w.podSC == nil && v == false {
		return
	}
	w.ensurePodSC()
	w.podSC.HostIPC = v
}


SELinuxOptions

func (w *podSecurityContextWrapper) SELinuxOptions() *api.SELinuxOptions {
	if w.podSC == nil {
		return nil
	}
	return w.podSC.SELinuxOptions
}


SetSELinuxOptions

func (w *podSecurityContextWrapper) SetSELinuxOptions(v *api.SELinuxOptions) {
	if w.podSC == nil && v == nil {
		return
	}
	w.ensurePodSC()
	w.podSC.SELinuxOptions = v
}


RunAsUser

func (w *podSecurityContextWrapper) RunAsUser() *int64 {
	if w.podSC == nil {
		return nil
	}
	return w.podSC.RunAsUser
}


SetRunAsUser

func (w *podSecurityContextWrapper) SetRunAsUser(v *int64) {
	if w.podSC == nil && v == nil {
		return
	}
	w.ensurePodSC()
	w.podSC.RunAsUser = v
}


RunAsGroup

func (w *podSecurityContextWrapper) RunAsGroup() *int64 {
	if w.podSC == nil {
		return nil
	}
	return w.podSC.RunAsGroup
}


SetRunAsGroup

func (w *podSecurityContextWrapper) SetRunAsGroup(v *int64) {
	if w.podSC == nil && v == nil {
		return
	}
	w.ensurePodSC()
	w.podSC.RunAsGroup = v
}


RunAsNonRoot

func (w *podSecurityContextWrapper) RunAsNonRoot() *bool {
	if w.podSC == nil {
		return nil
	}
	return w.podSC.RunAsNonRoot
}


SetRunAsNonRoot

func (w *podSecurityContextWrapper) SetRunAsNonRoot(v *bool) {
	if w.podSC == nil && v == nil {
		return
	}
	w.ensurePodSC()
	w.podSC.RunAsNonRoot = v
}


SeccompProfile

func (w *podSecurityContextWrapper) SeccompProfile() *api.SeccompProfile {
	if w.podSC == nil {
		return nil
	}
	return w.podSC.SeccompProfile
}


SetSeccompProfile

func (w *podSecurityContextWrapper) SetSeccompProfile(p *api.SeccompProfile) {
	if w.podSC == nil && p == nil {
		return
	}
	w.ensurePodSC()
	w.podSC.SeccompProfile = p
}


SupplementalGroups

func (w *podSecurityContextWrapper) SupplementalGroups() []int64 {
	if w.podSC == nil {
		return nil
	}
	return w.podSC.SupplementalGroups
}


SetSupplementalGroups

func (w *podSecurityContextWrapper) SetSupplementalGroups(v []int64) {
	if w.podSC == nil && len(v) == 0 {
		return
	}
	w.ensurePodSC()
	if len(v) == 0 && len(w.podSC.SupplementalGroups) == 0 {
		return
	}
	w.podSC.SupplementalGroups = v
}


FSGroup

func (w *podSecurityContextWrapper) FSGroup() *int64 {
	if w.podSC == nil {
		return nil
	}
	return w.podSC.FSGroup
}


SetFSGroup

func (w *podSecurityContextWrapper) SetFSGroup(v *int64) {
	if w.podSC == nil && v == nil {
		return
	}
	w.ensurePodSC()
	w.podSC.FSGroup = v
}


NewContainerSecurityContextAccessor

// NewContainerSecurityContextAccessor returns an accessor for the provided container security context
// May be initialized with a nil SecurityContext
func NewContainerSecurityContextAccessor(containerSC *api.SecurityContext) ContainerSecurityContextAccessor {
	return &containerSecurityContextWrapper{containerSC: containerSC}
}


NewContainerSecurityContextMutator

// NewContainerSecurityContextMutator returns a mutator for the provided container security context
// May be initialized with a nil SecurityContext
func NewContainerSecurityContextMutator(containerSC *api.SecurityContext) ContainerSecurityContextMutator {
	return &containerSecurityContextWrapper{containerSC: containerSC}
}


ContainerSecurityContext

func (w *containerSecurityContextWrapper) ContainerSecurityContext() *api.SecurityContext {
	return w.containerSC
}


ensureContainerSC

func (w *containerSecurityContextWrapper) ensureContainerSC() {
	if w.containerSC == nil {
		w.containerSC = &api.SecurityContext{}
	}
}


Capabilities

func (w *containerSecurityContextWrapper) Capabilities() *api.Capabilities {
	if w.containerSC == nil {
		return nil
	}
	return w.containerSC.Capabilities
}


SetCapabilities

func (w *containerSecurityContextWrapper) SetCapabilities(v *api.Capabilities) {
	if w.containerSC == nil && v == nil {
		return
	}
	w.ensureContainerSC()
	w.containerSC.Capabilities = v
}


Privileged

func (w *containerSecurityContextWrapper) Privileged() *bool {
	if w.containerSC == nil {
		return nil
	}
	return w.containerSC.Privileged
}


SetPrivileged

func (w *containerSecurityContextWrapper) SetPrivileged(v *bool) {
	if w.containerSC == nil && v == nil {
		return
	}
	w.ensureContainerSC()
	w.containerSC.Privileged = v
}


ProcMount

func (w *containerSecurityContextWrapper) ProcMount() api.ProcMountType {
	if w.containerSC == nil {
		return api.DefaultProcMount
	}
	if w.containerSC.ProcMount == nil {
		return api.DefaultProcMount
	}
	return *w.containerSC.ProcMount
}


SELinuxOptions

func (w *containerSecurityContextWrapper) SELinuxOptions() *api.SELinuxOptions {
	if w.containerSC == nil {
		return nil
	}
	return w.containerSC.SELinuxOptions
}


SetSELinuxOptions

func (w *containerSecurityContextWrapper) SetSELinuxOptions(v *api.SELinuxOptions) {
	if w.containerSC == nil && v == nil {
		return
	}
	w.ensureContainerSC()
	w.containerSC.SELinuxOptions = v
}


RunAsUser

func (w *containerSecurityContextWrapper) RunAsUser() *int64 {
	if w.containerSC == nil {
		return nil
	}
	return w.containerSC.RunAsUser
}


SetRunAsUser

func (w *containerSecurityContextWrapper) SetRunAsUser(v *int64) {
	if w.containerSC == nil && v == nil {
		return
	}
	w.ensureContainerSC()
	w.containerSC.RunAsUser = v
}


RunAsGroup

func (w *containerSecurityContextWrapper) RunAsGroup() *int64 {
	if w.containerSC == nil {
		return nil
	}
	return w.containerSC.RunAsGroup
}


SetRunAsGroup

func (w *containerSecurityContextWrapper) SetRunAsGroup(v *int64) {
	if w.containerSC == nil && v == nil {
		return
	}
	w.ensureContainerSC()
	w.containerSC.RunAsGroup = v
}


RunAsNonRoot

func (w *containerSecurityContextWrapper) RunAsNonRoot() *bool {
	if w.containerSC == nil {
		return nil
	}
	return w.containerSC.RunAsNonRoot
}


SetRunAsNonRoot

func (w *containerSecurityContextWrapper) SetRunAsNonRoot(v *bool) {
	if w.containerSC == nil && v == nil {
		return
	}
	w.ensureContainerSC()
	w.containerSC.RunAsNonRoot = v
}


ReadOnlyRootFilesystem

func (w *containerSecurityContextWrapper) ReadOnlyRootFilesystem() *bool {
	if w.containerSC == nil {
		return nil
	}
	return w.containerSC.ReadOnlyRootFilesystem
}


SetReadOnlyRootFilesystem

func (w *containerSecurityContextWrapper) SetReadOnlyRootFilesystem(v *bool) {
	if w.containerSC == nil && v == nil {
		return
	}
	w.ensureContainerSC()
	w.containerSC.ReadOnlyRootFilesystem = v
}


SeccompProfile

func (w *containerSecurityContextWrapper) SeccompProfile() *api.SeccompProfile {
	if w.containerSC == nil {
		return nil
	}
	return w.containerSC.SeccompProfile
}


SetSeccompProfile

func (w *containerSecurityContextWrapper) SetSeccompProfile(p *api.SeccompProfile) {
	if w.containerSC == nil && p == nil {
		return
	}
	w.ensureContainerSC()
	w.containerSC.SeccompProfile = p
}


AllowPrivilegeEscalation

func (w *containerSecurityContextWrapper) AllowPrivilegeEscalation() *bool {
	if w.containerSC == nil {
		return nil
	}
	return w.containerSC.AllowPrivilegeEscalation
}


SetAllowPrivilegeEscalation

func (w *containerSecurityContextWrapper) SetAllowPrivilegeEscalation(v *bool) {
	if w.containerSC == nil && v == nil {
		return
	}
	w.ensureContainerSC()
	w.containerSC.AllowPrivilegeEscalation = v
}


// NewEffectiveContainerSecurityContextAccessor returns an accessor for reading effective values
// for the provided pod security context and container security context
func NewEffectiveContainerSecurityContextAccessor(podSC PodSecurityContextAccessor, containerSC ContainerSecurityContextMutator) ContainerSecurityContextAccessor {
	return &effectiveContainerSecurityContextWrapper{podSC: podSC, containerSC: containerSC}
}


// NewEffectiveContainerSecurityContextMutator returns a mutator for reading and writing effective values
// for the provided pod security context and container security context
func NewEffectiveContainerSecurityContextMutator(podSC PodSecurityContextAccessor, containerSC ContainerSecurityContextMutator) ContainerSecurityContextMutator {
	return &effectiveContainerSecurityContextWrapper{podSC: podSC, containerSC: containerSC}
}


ContainerSecurityContext

func (w *effectiveContainerSecurityContextWrapper) ContainerSecurityContext() *api.SecurityContext {
	return w.containerSC.ContainerSecurityContext()
}


Capabilities

func (w *effectiveContainerSecurityContextWrapper) Capabilities() *api.Capabilities {
	return w.containerSC.Capabilities()
}


SetCapabilities

func (w *effectiveContainerSecurityContextWrapper) SetCapabilities(v *api.Capabilities) {
	if !reflect.DeepEqual(w.Capabilities(), v) {
		w.containerSC.SetCapabilities(v)
	}
}


Privileged

func (w *effectiveContainerSecurityContextWrapper) Privileged() *bool {
	return w.containerSC.Privileged()
}


SetPrivileged

func (w *effectiveContainerSecurityContextWrapper) SetPrivileged(v *bool) {
	if !reflect.DeepEqual(w.Privileged(), v) {
		w.containerSC.SetPrivileged(v)
	}
}


ProcMount

func (w *effectiveContainerSecurityContextWrapper) ProcMount() api.ProcMountType {
	return w.containerSC.ProcMount()
}


SELinuxOptions

func (w *effectiveContainerSecurityContextWrapper) SELinuxOptions() *api.SELinuxOptions {
	if v := w.containerSC.SELinuxOptions(); v != nil {
		return v
	}
	return w.podSC.SELinuxOptions()
}


SetSELinuxOptions

func (w *effectiveContainerSecurityContextWrapper) SetSELinuxOptions(v *api.SELinuxOptions) {
	if !reflect.DeepEqual(w.SELinuxOptions(), v) {
		w.containerSC.SetSELinuxOptions(v)
	}
}


RunAsUser

func (w *effectiveContainerSecurityContextWrapper) RunAsUser() *int64 {
	if v := w.containerSC.RunAsUser(); v != nil {
		return v
	}
	return w.podSC.RunAsUser()
}


SetRunAsUser

func (w *effectiveContainerSecurityContextWrapper) SetRunAsUser(v *int64) {
	if !reflect.DeepEqual(w.RunAsUser(), v) {
		w.containerSC.SetRunAsUser(v)
	}
}


RunAsGroup

func (w *effectiveContainerSecurityContextWrapper) RunAsGroup() *int64 {
	if v := w.containerSC.RunAsGroup(); v != nil {
		return v
	}
	return w.podSC.RunAsGroup()
}


SetRunAsGroup

func (w *effectiveContainerSecurityContextWrapper) SetRunAsGroup(v *int64) {
	if !reflect.DeepEqual(w.RunAsGroup(), v) {
		w.containerSC.SetRunAsGroup(v)
	}
}


RunAsNonRoot

func (w *effectiveContainerSecurityContextWrapper) RunAsNonRoot() *bool {
	if v := w.containerSC.RunAsNonRoot(); v != nil {
		return v
	}
	return w.podSC.RunAsNonRoot()
}


SetRunAsNonRoot

func (w *effectiveContainerSecurityContextWrapper) SetRunAsNonRoot(v *bool) {
	if !reflect.DeepEqual(w.RunAsNonRoot(), v) {
		w.containerSC.SetRunAsNonRoot(v)
	}
}


ReadOnlyRootFilesystem

func (w *effectiveContainerSecurityContextWrapper) ReadOnlyRootFilesystem() *bool {
	return w.containerSC.ReadOnlyRootFilesystem()
}


SetReadOnlyRootFilesystem

func (w *effectiveContainerSecurityContextWrapper) SetReadOnlyRootFilesystem(v *bool) {
	if !reflect.DeepEqual(w.ReadOnlyRootFilesystem(), v) {
		w.containerSC.SetReadOnlyRootFilesystem(v)
	}
}


SeccompProfile

func (w *effectiveContainerSecurityContextWrapper) SeccompProfile() *api.SeccompProfile {
	return w.containerSC.SeccompProfile()
}


SetSeccompProfile

func (w *effectiveContainerSecurityContextWrapper) SetSeccompProfile(p *api.SeccompProfile) {
	if !reflect.DeepEqual(w.SeccompProfile(), p) {
		w.containerSC.SetSeccompProfile(p)
	}
}


AllowPrivilegeEscalation

func (w *effectiveContainerSecurityContextWrapper) AllowPrivilegeEscalation() *bool {
	return w.containerSC.AllowPrivilegeEscalation()
}


SetAllowPrivilegeEscalation

func (w *effectiveContainerSecurityContextWrapper) SetAllowPrivilegeEscalation(v *bool) {
	if !reflect.DeepEqual(w.AllowPrivilegeEscalation(), v) {
		w.containerSC.SetAllowPrivilegeEscalation(v)
	}
}


HasWindowsHostProcessRequest

// HasWindowsHostProcessRequest returns true if container should run as HostProcess container,
// taking into account nils
func HasWindowsHostProcessRequest(pod *v1.Pod, container *v1.Container) bool {
	effectiveSc := DetermineEffectiveSecurityContext(pod, container)

	if effectiveSc.WindowsOptions == nil {
		return false
	}
	if effectiveSc.WindowsOptions.HostProcess == nil {
		return false
	}
	return *effectiveSc.WindowsOptions.HostProcess
}


DetermineEffectiveSecurityContext

// DetermineEffectiveSecurityContext returns a synthesized SecurityContext for reading effective configurations
// from the provided pod's and container's security context. Container's fields take precedence in cases where both
// are set
func DetermineEffectiveSecurityContext(pod *v1.Pod, container *v1.Container) *v1.SecurityContext {
	effectiveSc := securityContextFromPodSecurityContext(pod)
	containerSc := container.SecurityContext

	if effectiveSc == nil && containerSc == nil {
		return &v1.SecurityContext{}
	}
	if effectiveSc != nil && containerSc == nil {
		return effectiveSc
	}
	if effectiveSc == nil && containerSc != nil {
		return containerSc
	}

	if containerSc.SELinuxOptions != nil {
		effectiveSc.SELinuxOptions = new(v1.SELinuxOptions)
		*effectiveSc.SELinuxOptions = *containerSc.SELinuxOptions
	}

	if containerSc.WindowsOptions != nil {
		// only override fields that are set at the container level, not the whole thing
		if effectiveSc.WindowsOptions == nil {
			effectiveSc.WindowsOptions = &v1.WindowsSecurityContextOptions{}
		}
		if containerSc.WindowsOptions.GMSACredentialSpecName != nil || containerSc.WindowsOptions.GMSACredentialSpec != nil {
			// both GMSA fields go hand in hand
			effectiveSc.WindowsOptions.GMSACredentialSpecName = containerSc.WindowsOptions.GMSACredentialSpecName
			effectiveSc.WindowsOptions.GMSACredentialSpec = containerSc.WindowsOptions.GMSACredentialSpec
		}
		if containerSc.WindowsOptions.RunAsUserName != nil {
			effectiveSc.WindowsOptions.RunAsUserName = containerSc.WindowsOptions.RunAsUserName
		}
		if containerSc.WindowsOptions.HostProcess != nil {
			effectiveSc.WindowsOptions.HostProcess = containerSc.WindowsOptions.HostProcess
		}
	}

	if containerSc.Capabilities != nil {
		effectiveSc.Capabilities = new(v1.Capabilities)
		*effectiveSc.Capabilities = *containerSc.Capabilities
	}

	if containerSc.Privileged != nil {
		effectiveSc.Privileged = new(bool)
		*effectiveSc.Privileged = *containerSc.Privileged
	}

	if containerSc.RunAsUser != nil {
		effectiveSc.RunAsUser = new(int64)
		*effectiveSc.RunAsUser = *containerSc.RunAsUser
	}

	if containerSc.RunAsGroup != nil {
		effectiveSc.RunAsGroup = new(int64)
		*effectiveSc.RunAsGroup = *containerSc.RunAsGroup
	}

	if containerSc.RunAsNonRoot != nil {
		effectiveSc.RunAsNonRoot = new(bool)
		*effectiveSc.RunAsNonRoot = *containerSc.RunAsNonRoot
	}

	if containerSc.ReadOnlyRootFilesystem != nil {
		effectiveSc.ReadOnlyRootFilesystem = new(bool)
		*effectiveSc.ReadOnlyRootFilesystem = *containerSc.ReadOnlyRootFilesystem
	}

	if containerSc.AllowPrivilegeEscalation != nil {
		effectiveSc.AllowPrivilegeEscalation = new(bool)
		*effectiveSc.AllowPrivilegeEscalation = *containerSc.AllowPrivilegeEscalation
	}

	if containerSc.ProcMount != nil {
		effectiveSc.ProcMount = new(v1.ProcMountType)
		*effectiveSc.ProcMount = *containerSc.ProcMount
	}

	return effectiveSc
}


DetermineEffectiveRunAsUser

// DetermineEffectiveRunAsUser returns a pointer of UID from the provided pod's
// and container's security context and a bool value to indicate if it is absent.
// Container's runAsUser take precedence in cases where both are set.
func DetermineEffectiveRunAsUser(pod *v1.Pod, container *v1.Container) (*int64, bool) {
	var runAsUser *int64
	if pod.Spec.SecurityContext != nil && pod.Spec.SecurityContext.RunAsUser != nil {
		runAsUser = new(int64)
		*runAsUser = *pod.Spec.SecurityContext.RunAsUser
	}
	if container.SecurityContext != nil && container.SecurityContext.RunAsUser != nil {
		runAsUser = new(int64)
		*runAsUser = *container.SecurityContext.RunAsUser
	}
	if runAsUser == nil {
		return nil, false
	}
	return runAsUser, true
}


securityContextFromPodSecurityContext

func securityContextFromPodSecurityContext(pod *v1.Pod) *v1.SecurityContext {
	if pod.Spec.SecurityContext == nil {
		return nil
	}

	synthesized := &v1.SecurityContext{}

	if pod.Spec.SecurityContext.SELinuxOptions != nil {
		synthesized.SELinuxOptions = &v1.SELinuxOptions{}
		*synthesized.SELinuxOptions = *pod.Spec.SecurityContext.SELinuxOptions
	}

	if pod.Spec.SecurityContext.WindowsOptions != nil {
		synthesized.WindowsOptions = &v1.WindowsSecurityContextOptions{}
		*synthesized.WindowsOptions = *pod.Spec.SecurityContext.WindowsOptions
	}

	if pod.Spec.SecurityContext.RunAsUser != nil {
		synthesized.RunAsUser = new(int64)
		*synthesized.RunAsUser = *pod.Spec.SecurityContext.RunAsUser
	}

	if pod.Spec.SecurityContext.RunAsGroup != nil {
		synthesized.RunAsGroup = new(int64)
		*synthesized.RunAsGroup = *pod.Spec.SecurityContext.RunAsGroup
	}

	if pod.Spec.SecurityContext.RunAsNonRoot != nil {
		synthesized.RunAsNonRoot = new(bool)
		*synthesized.RunAsNonRoot = *pod.Spec.SecurityContext.RunAsNonRoot
	}

	return synthesized
}


AddNoNewPrivileges

// AddNoNewPrivileges returns if we should add the no_new_privs option.
func AddNoNewPrivileges(sc *v1.SecurityContext) bool {
	if sc == nil {
		return false
	}

	// handle the case where the user did not set the default and did not explicitly set allowPrivilegeEscalation
	if sc.AllowPrivilegeEscalation == nil {
		return false
	}

	// handle the case where defaultAllowPrivilegeEscalation is false or the user explicitly set allowPrivilegeEscalation to true/false
	return !*sc.AllowPrivilegeEscalation
}


ConvertToRuntimeMaskedPaths

// ConvertToRuntimeMaskedPaths converts the ProcMountType to the specified or default
// masked paths.
func ConvertToRuntimeMaskedPaths(opt *v1.ProcMountType) []string {
	if opt != nil && *opt == v1.UnmaskedProcMount {
		// Unmasked proc mount should have no paths set as masked.
		return []string{}
	}

	// Otherwise, add the default masked paths to the runtime security context.
	return defaultMaskedPaths
}


ConvertToRuntimeReadonlyPaths

// ConvertToRuntimeReadonlyPaths converts the ProcMountType to the specified or default
// readonly paths.
func ConvertToRuntimeReadonlyPaths(opt *v1.ProcMountType) []string {
	if opt != nil && *opt == v1.UnmaskedProcMount {
		// Unmasked proc mount should have no paths set as readonly.
		return []string{}
	}

	// Otherwise, add the default readonly paths to the runtime security context.
	return defaultReadonlyPaths
}


Claims

func Claims(sa core.ServiceAccount, pod *core.Pod, secret *core.Secret, expirationSeconds, warnafter int64, audience []string) (*jwt.Claims, interface{}) {
	now := now()
	sc := &jwt.Claims{
		Subject:   apiserverserviceaccount.MakeUsername(sa.Namespace, sa.Name),
		Audience:  jwt.Audience(audience),
		IssuedAt:  jwt.NewNumericDate(now),
		NotBefore: jwt.NewNumericDate(now),
		Expiry:    jwt.NewNumericDate(now.Add(time.Duration(expirationSeconds) * time.Second)),
	}
	pc := &privateClaims{
		Kubernetes: kubernetes{
			Namespace: sa.Namespace,
			Svcacct: ref{
				Name: sa.Name,
				UID:  string(sa.UID),
			},
		},
	}
	switch {
	case pod != nil:
		pc.Kubernetes.Pod = &ref{
			Name: pod.Name,
			UID:  string(pod.UID),
		}
	case secret != nil:
		pc.Kubernetes.Secret = &ref{
			Name: secret.Name,
			UID:  string(secret.UID),
		}
	}

	if warnafter != 0 {
		pc.Kubernetes.WarnAfter = jwt.NewNumericDate(now.Add(time.Duration(warnafter) * time.Second))
	}

	return sc, pc
}


NewValidator

func NewValidator(getter ServiceAccountTokenGetter) Validator {
	return &validator{
		getter: getter,
	}
}


Validate

func (v *validator) Validate(ctx context.Context, _ string, public *jwt.Claims, privateObj interface{}) (*apiserverserviceaccount.ServiceAccountInfo, error) {
	private, ok := privateObj.(*privateClaims)
	if !ok {
		klog.Errorf("service account jwt validator expected private claim of type *privateClaims but got: %T", privateObj)
		return nil, errors.New("service account token claims could not be validated due to unexpected private claim")
	}
	nowTime := now()
	err := public.Validate(jwt.Expected{
		Time: nowTime,
	})
	switch err {
	case nil:
		// successful validation

	case jwt.ErrExpired:
		return nil, errors.New("service account token has expired")

	case jwt.ErrNotValidYet:
		return nil, errors.New("service account token is not valid yet")

	case jwt.ErrIssuedInTheFuture:
		return nil, errors.New("service account token is issued in the future")

	// our current use of jwt.Expected above should make these cases impossible to hit
	case jwt.ErrInvalidAudience, jwt.ErrInvalidID, jwt.ErrInvalidIssuer, jwt.ErrInvalidSubject:
		klog.Errorf("service account token claim validation got unexpected validation failure: %v", err)
		return nil, fmt.Errorf("service account token claims could not be validated: %w", err) // safe to pass these errors back to the user

	default:
		klog.Errorf("service account token claim validation got unexpected error type: %T", err)                         // avoid leaking unexpected information into the logs
		return nil, errors.New("service account token claims could not be validated due to unexpected validation error") // return an opaque error
	}

	// consider things deleted prior to now()-leeway to be invalid
	invalidIfDeletedBefore := nowTime.Add(-jwt.DefaultLeeway)
	namespace := private.Kubernetes.Namespace
	saref := private.Kubernetes.Svcacct
	podref := private.Kubernetes.Pod
	secref := private.Kubernetes.Secret
	// Make sure service account still exists (name and UID)
	serviceAccount, err := v.getter.GetServiceAccount(namespace, saref.Name)
	if err != nil {
		klog.V(4).Infof("Could not retrieve service account %s/%s: %v", namespace, saref.Name, err)
		return nil, err
	}
	if serviceAccount.DeletionTimestamp != nil && serviceAccount.DeletionTimestamp.Time.Before(invalidIfDeletedBefore) {
		klog.V(4).Infof("Service account has been deleted %s/%s", namespace, saref.Name)
		return nil, fmt.Errorf("service account %s/%s has been deleted", namespace, saref.Name)
	}
	if string(serviceAccount.UID) != saref.UID {
		klog.V(4).Infof("Service account UID no longer matches %s/%s: %q != %q", namespace, saref.Name, string(serviceAccount.UID), saref.UID)
		return nil, fmt.Errorf("service account UID (%s) does not match claim (%s)", serviceAccount.UID, saref.UID)
	}

	if secref != nil {
		// Make sure token hasn't been invalidated by deletion of the secret
		secret, err := v.getter.GetSecret(namespace, secref.Name)
		if err != nil {
			klog.V(4).Infof("Could not retrieve bound secret %s/%s for service account %s/%s: %v", namespace, secref.Name, namespace, saref.Name, err)
			return nil, errors.New("service account token has been invalidated")
		}
		if secret.DeletionTimestamp != nil && secret.DeletionTimestamp.Time.Before(invalidIfDeletedBefore) {
			klog.V(4).Infof("Bound secret is deleted and awaiting removal: %s/%s for service account %s/%s", namespace, secref.Name, namespace, saref.Name)
			return nil, errors.New("service account token has been invalidated")
		}
		if secref.UID != string(secret.UID) {
			klog.V(4).Infof("Secret UID no longer matches %s/%s: %q != %q", namespace, secref.Name, string(secret.UID), secref.UID)
			return nil, fmt.Errorf("secret UID (%s) does not match service account secret ref claim (%s)", secret.UID, secref.UID)
		}
	}

	var podName, podUID string
	if podref != nil {
		// Make sure token hasn't been invalidated by deletion of the pod
		pod, err := v.getter.GetPod(namespace, podref.Name)
		if err != nil {
			klog.V(4).Infof("Could not retrieve bound pod %s/%s for service account %s/%s: %v", namespace, podref.Name, namespace, saref.Name, err)
			return nil, errors.New("service account token has been invalidated")
		}
		if pod.DeletionTimestamp != nil && pod.DeletionTimestamp.Time.Before(invalidIfDeletedBefore) {
			klog.V(4).Infof("Bound pod is deleted and awaiting removal: %s/%s for service account %s/%s", namespace, podref.Name, namespace, saref.Name)
			return nil, errors.New("service account token has been invalidated")
		}
		if podref.UID != string(pod.UID) {
			klog.V(4).Infof("Pod UID no longer matches %s/%s: %q != %q", namespace, podref.Name, string(pod.UID), podref.UID)
			return nil, fmt.Errorf("pod UID (%s) does not match service account pod ref claim (%s)", pod.UID, podref.UID)
		}
		podName = podref.Name
		podUID = podref.UID
	}

	// Check special 'warnafter' field for projected service account token transition.
	warnafter := private.Kubernetes.WarnAfter
	if warnafter != nil && *warnafter != 0 {
		if nowTime.After(warnafter.Time()) {
			secondsAfterWarn := nowTime.Unix() - warnafter.Time().Unix()
			auditInfo := fmt.Sprintf("subject: %s, seconds after warning threshold: %d", public.Subject, secondsAfterWarn)
			audit.AddAuditAnnotation(ctx, "authentication.k8s.io/stale-token", auditInfo)
			staleTokensTotal.WithContext(ctx).Inc()
		} else {
			validTokensTotal.WithContext(ctx).Inc()
		}
	}

	return &apiserverserviceaccount.ServiceAccountInfo{
		Namespace: private.Kubernetes.Namespace,
		Name:      private.Kubernetes.Svcacct.Name,
		UID:       private.Kubernetes.Svcacct.UID,
		PodName:   podName,
		PodUID:    podUID,
	}, nil
}


NewPrivateClaims

func (v *validator) NewPrivateClaims() interface{} {
	return &privateClaims{}
}


JWTTokenGenerator

// JWTTokenGenerator returns a TokenGenerator that generates signed JWT tokens, using the given privateKey.
// privateKey is a PEM-encoded byte array of a private RSA key.
func JWTTokenGenerator(iss string, privateKey interface{}) (TokenGenerator, error) {
	var signer jose.Signer
	var err error
	switch pk := privateKey.(type) {
	case *rsa.PrivateKey:
		signer, err = signerFromRSAPrivateKey(pk)
		if err != nil {
			return nil, fmt.Errorf("could not generate signer for RSA keypair: %v", err)
		}
	case *ecdsa.PrivateKey:
		signer, err = signerFromECDSAPrivateKey(pk)
		if err != nil {
			return nil, fmt.Errorf("could not generate signer for ECDSA keypair: %v", err)
		}
	case jose.OpaqueSigner:
		signer, err = signerFromOpaqueSigner(pk)
		if err != nil {
			return nil, fmt.Errorf("could not generate signer for OpaqueSigner: %v", err)
		}
	default:
		return nil, fmt.Errorf("unknown private key type %T, must be *rsa.PrivateKey, *ecdsa.PrivateKey, or jose.OpaqueSigner", privateKey)
	}

	return &jwtTokenGenerator{
		iss:    iss,
		signer: signer,
	}, nil
}


keyIDFromPublicKey

// keyIDFromPublicKey derives a key ID non-reversibly from a public key.
//
// The Key ID is field on a given on JWTs and JWKs that help relying parties
// pick the correct key for verification when the identity party advertises
// multiple keys.
//
// Making the derivation non-reversible makes it impossible for someone to
// accidentally obtain the real key from the key ID and use it for token
// validation.
func keyIDFromPublicKey(publicKey interface{}) (string, error) {
	publicKeyDERBytes, err := x509.MarshalPKIXPublicKey(publicKey)
	if err != nil {
		return "", fmt.Errorf("failed to serialize public key to DER format: %v", err)
	}

	hasher := crypto.SHA256.New()
	hasher.Write(publicKeyDERBytes)
	publicKeyDERHash := hasher.Sum(nil)

	keyID := base64.RawURLEncoding.EncodeToString(publicKeyDERHash)

	return keyID, nil
}


signerFromRSAPrivateKey

func signerFromRSAPrivateKey(keyPair *rsa.PrivateKey) (jose.Signer, error) {
	keyID, err := keyIDFromPublicKey(&keyPair.PublicKey)
	if err != nil {
		return nil, fmt.Errorf("failed to derive keyID: %v", err)
	}

	// IMPORTANT: If this function is updated to support additional key sizes,
	// algorithmForPublicKey in serviceaccount/openidmetadata.go must also be
	// updated to support the same key sizes. Today we only support RS256.

	// Wrap the RSA keypair in a JOSE JWK with the designated key ID.
	privateJWK := &jose.JSONWebKey{
		Algorithm: string(jose.RS256),
		Key:       keyPair,
		KeyID:     keyID,
		Use:       "sig",
	}

	signer, err := jose.NewSigner(
		jose.SigningKey{
			Algorithm: jose.RS256,
			Key:       privateJWK,
		},
		nil,
	)

	if err != nil {
		return nil, fmt.Errorf("failed to create signer: %v", err)
	}

	return signer, nil
}


signerFromECDSAPrivateKey

func signerFromECDSAPrivateKey(keyPair *ecdsa.PrivateKey) (jose.Signer, error) {
	var alg jose.SignatureAlgorithm
	switch keyPair.Curve {
	case elliptic.P256():
		alg = jose.ES256
	case elliptic.P384():
		alg = jose.ES384
	case elliptic.P521():
		alg = jose.ES512
	default:
		return nil, fmt.Errorf("unknown private key curve, must be 256, 384, or 521")
	}

	keyID, err := keyIDFromPublicKey(&keyPair.PublicKey)
	if err != nil {
		return nil, fmt.Errorf("failed to derive keyID: %v", err)
	}

	// Wrap the ECDSA keypair in a JOSE JWK with the designated key ID.
	privateJWK := &jose.JSONWebKey{
		Algorithm: string(alg),
		Key:       keyPair,
		KeyID:     keyID,
		Use:       "sig",
	}

	signer, err := jose.NewSigner(
		jose.SigningKey{
			Algorithm: alg,
			Key:       privateJWK,
		},
		nil,
	)
	if err != nil {
		return nil, fmt.Errorf("failed to create signer: %v", err)
	}

	return signer, nil
}


signerFromOpaqueSigner

func signerFromOpaqueSigner(opaqueSigner jose.OpaqueSigner) (jose.Signer, error) {
	alg := jose.SignatureAlgorithm(opaqueSigner.Public().Algorithm)

	signer, err := jose.NewSigner(
		jose.SigningKey{
			Algorithm: alg,
			Key: &jose.JSONWebKey{
				Algorithm: string(alg),
				Key:       opaqueSigner,
				KeyID:     opaqueSigner.Public().KeyID,
				Use:       "sig",
			},
		},
		nil,
	)
	if err != nil {
		return nil, fmt.Errorf("failed to create signer: %v", err)
	}

	return signer, nil
}


GenerateToken

func (j *jwtTokenGenerator) GenerateToken(claims *jwt.Claims, privateClaims interface{}) (string, error) {
	// claims are applied in reverse precedence
	return jwt.Signed(j.signer).
		Claims(privateClaims).
		Claims(claims).
		Claims(&jwt.Claims{
			Issuer: j.iss,
		}).
		CompactSerialize()
}


JWTTokenAuthenticator

// JWTTokenAuthenticator authenticates tokens as JWT tokens produced by JWTTokenGenerator
// Token signatures are verified using each of the given public keys until one works (allowing key rotation)
// If lookup is true, the service account and secret referenced as claims inside the token are retrieved and verified with the provided ServiceAccountTokenGetter
func JWTTokenAuthenticator(issuers []string, keys []interface{}, implicitAuds authenticator.Audiences, validator Validator) authenticator.Token {
	issuersMap := make(map[string]bool)
	for _, issuer := range issuers {
		issuersMap[issuer] = true
	}
	return &jwtTokenAuthenticator{
		issuers:      issuersMap,
		keys:         keys,
		implicitAuds: implicitAuds,
		validator:    validator,
	}
}


AuthenticateToken

func (j *jwtTokenAuthenticator) AuthenticateToken(ctx context.Context, tokenData string) (*authenticator.Response, bool, error) {
	if !j.hasCorrectIssuer(tokenData) {
		return nil, false, nil
	}

	tok, err := jwt.ParseSigned(tokenData)
	if err != nil {
		return nil, false, nil
	}

	public := &jwt.Claims{}
	private := j.validator.NewPrivateClaims()

	// TODO: Pick the key that has the same key ID as `tok`, if one exists.
	var (
		found   bool
		errlist []error
	)
	for _, key := range j.keys {
		if err := tok.Claims(key, public, private); err != nil {
			errlist = append(errlist, err)
			continue
		}
		found = true
		break
	}

	if !found {
		return nil, false, utilerrors.NewAggregate(errlist)
	}

	tokenAudiences := authenticator.Audiences(public.Audience)
	if len(tokenAudiences) == 0 {
		// only apiserver audiences are allowed for legacy tokens
		audit.AddAuditAnnotation(ctx, "authentication.k8s.io/legacy-token", public.Subject)
		legacyTokensTotal.WithContext(ctx).Inc()
		tokenAudiences = j.implicitAuds
	}

	requestedAudiences, ok := authenticator.AudiencesFrom(ctx)
	if !ok {
		// default to apiserver audiences
		requestedAudiences = j.implicitAuds
	}

	auds := authenticator.Audiences(tokenAudiences).Intersect(requestedAudiences)
	if len(auds) == 0 && len(j.implicitAuds) != 0 {
		return nil, false, fmt.Errorf("token audiences %q is invalid for the target audiences %q", tokenAudiences, requestedAudiences)
	}

	// If we get here, we have a token with a recognized signature and
	// issuer string.
	sa, err := j.validator.Validate(ctx, tokenData, public, private)
	if err != nil {
		return nil, false, err
	}

	return &authenticator.Response{
		User:      sa.UserInfo(),
		Audiences: auds,
	}, true, nil
}


hasCorrectIssuer

// hasCorrectIssuer returns true if tokenData is a valid JWT in compact
// serialization format and the "iss" claim matches the iss field of this token
// authenticator, and otherwise returns false.
//
// Note: go-jose currently does not allow access to unverified JWS payloads.
// See https://github.com/square/go-jose/issues/169
func (j *jwtTokenAuthenticator) hasCorrectIssuer(tokenData string) bool {
	parts := strings.Split(tokenData, ".")
	if len(parts) != 3 {
		return false
	}
	payload, err := base64.RawURLEncoding.DecodeString(parts[1])
	if err != nil {
		return false
	}
	claims := struct {
		// WARNING: this JWT is not verified. Do not trust these claims.
		Issuer string `json:"iss"`
	}{}
	if err := json.Unmarshal(payload, &claims); err != nil {
		return false
	}
	return j.issuers[claims.Issuer]
}


LegacyClaims

func LegacyClaims(serviceAccount v1.ServiceAccount, secret v1.Secret) (*jwt.Claims, interface{}) {
	return &jwt.Claims{
			Subject: apiserverserviceaccount.MakeUsername(serviceAccount.Namespace, serviceAccount.Name),
		}, &legacyPrivateClaims{
			Namespace:          serviceAccount.Namespace,
			ServiceAccountName: serviceAccount.Name,
			ServiceAccountUID:  string(serviceAccount.UID),
			SecretName:         secret.Name,
		}
}


NewLegacyValidator

func NewLegacyValidator(lookup bool, getter ServiceAccountTokenGetter, secretsWriter typedv1core.SecretsGetter) (Validator, error) {
	if lookup && getter == nil {
		return nil, errors.New("ServiceAccountTokenGetter must be provided")
	}
	if lookup && secretsWriter == nil && utilfeature.DefaultFeatureGate.Enabled(kubefeatures.LegacyServiceAccountTokenTracking) {
		return nil, errors.New("SecretsWriter must be provided")
	}
	return &legacyValidator{
		lookup:        lookup,
		getter:        getter,
		secretsWriter: secretsWriter,
	}, nil
}


Validate

func (v *legacyValidator) Validate(ctx context.Context, tokenData string, public *jwt.Claims, privateObj interface{}) (*apiserverserviceaccount.ServiceAccountInfo, error) {
	private, ok := privateObj.(*legacyPrivateClaims)
	if !ok {
		klog.Errorf("jwt validator expected private claim of type *legacyPrivateClaims but got: %T", privateObj)
		return nil, errors.New("Token could not be validated.")
	}

	// Make sure the claims we need exist
	if len(public.Subject) == 0 {
		return nil, errors.New("sub claim is missing")
	}
	namespace := private.Namespace
	if len(namespace) == 0 {
		return nil, errors.New("namespace claim is missing")
	}
	secretName := private.SecretName
	if len(secretName) == 0 {
		return nil, errors.New("secretName claim is missing")
	}
	serviceAccountName := private.ServiceAccountName
	if len(serviceAccountName) == 0 {
		return nil, errors.New("serviceAccountName claim is missing")
	}
	serviceAccountUID := private.ServiceAccountUID
	if len(serviceAccountUID) == 0 {
		return nil, errors.New("serviceAccountUID claim is missing")
	}

	subjectNamespace, subjectName, err := apiserverserviceaccount.SplitUsername(public.Subject)
	if err != nil || subjectNamespace != namespace || subjectName != serviceAccountName {
		return nil, errors.New("sub claim is invalid")
	}

	if v.lookup {
		// Make sure token hasn't been invalidated by deletion of the secret
		secret, err := v.getter.GetSecret(namespace, secretName)
		if err != nil {
			klog.V(4).Infof("Could not retrieve token %s/%s for service account %s/%s: %v", namespace, secretName, namespace, serviceAccountName, err)
			return nil, errors.New("Token has been invalidated")
		}
		if secret.DeletionTimestamp != nil {
			klog.V(4).Infof("Token is deleted and awaiting removal: %s/%s for service account %s/%s", namespace, secretName, namespace, serviceAccountName)
			return nil, errors.New("Token has been invalidated")
		}
		if !bytes.Equal(secret.Data[v1.ServiceAccountTokenKey], []byte(tokenData)) {
			klog.V(4).Infof("Token contents no longer matches %s/%s for service account %s/%s", namespace, secretName, namespace, serviceAccountName)
			return nil, errors.New("Token does not match server's copy")
		}

		// Make sure service account still exists (name and UID)
		serviceAccount, err := v.getter.GetServiceAccount(namespace, serviceAccountName)
		if err != nil {
			klog.V(4).Infof("Could not retrieve service account %s/%s: %v", namespace, serviceAccountName, err)
			return nil, err
		}
		if serviceAccount.DeletionTimestamp != nil {
			klog.V(4).Infof("Service account has been deleted %s/%s", namespace, serviceAccountName)
			return nil, fmt.Errorf("ServiceAccount %s/%s has been deleted", namespace, serviceAccountName)
		}
		if string(serviceAccount.UID) != serviceAccountUID {
			klog.V(4).Infof("Service account UID no longer matches %s/%s: %q != %q", namespace, serviceAccountName, string(serviceAccount.UID), serviceAccountUID)
			return nil, fmt.Errorf("ServiceAccount UID (%s) does not match claim (%s)", serviceAccount.UID, serviceAccountUID)
		}

		if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.LegacyServiceAccountTokenTracking) {
			for _, ref := range serviceAccount.Secrets {
				if ref.Name == secret.Name {
					warning.AddWarning(ctx, "", "Use tokens from the TokenRequest API or manually created secret-based tokens instead of auto-generated secret-based tokens.")
					break
				}
			}
			now := time.Now().UTC()
			today := now.Format("2006-01-02")
			tomorrow := now.AddDate(0, 0, 1).Format("2006-01-02")
			lastUsed := secret.Labels[LastUsedLabelKey]
			if lastUsed != today && lastUsed != tomorrow {
				patchContent, err := json.Marshal(applyv1.Secret(secret.Name, secret.Namespace).WithLabels(map[string]string{LastUsedLabelKey: today}))
				if err != nil {
					klog.Errorf("Failed to marshal legacy service account token tracking labels, err: %v", err)
				} else {
					if _, err := v.secretsWriter.Secrets(namespace).Patch(ctx, secret.Name, types.MergePatchType, patchContent, metav1.PatchOptions{}); err != nil {
						klog.Errorf("Failed to label legacy service account token secret with last-used, err: %v", err)
					}
				}
			}
		}
	}

	return &apiserverserviceaccount.ServiceAccountInfo{
		Namespace: private.Namespace,
		Name:      private.ServiceAccountName,
		UID:       private.ServiceAccountUID,
	}, nil
}


NewPrivateClaims

func (v *legacyValidator) NewPrivateClaims() interface{} {
	return &legacyPrivateClaims{}
}


RegisterMetrics

func RegisterMetrics() {
	registerMetricsOnce.Do(func() {
		legacyregistry.MustRegister(legacyTokensTotal)
		legacyregistry.MustRegister(staleTokensTotal)
		legacyregistry.MustRegister(validTokensTotal)
	})
}


NewOpenIDMetadata

// NewOpenIDMetadata returns the pre-rendered JSON responses for the OIDC discovery
// endpoints, or an error if they could not be constructed. Callers should note
// that this function may perform additional validation on inputs that is not
// backwards-compatible with all command-line validation. The recommendation is
// to log the error and skip installing the OIDC discovery endpoints.
func NewOpenIDMetadata(issuerURL, jwksURI, defaultExternalAddress string, pubKeys []interface{}) (*OpenIDMetadata, error) {
	if issuerURL == "" {
		return nil, fmt.Errorf("empty issuer URL")
	}
	if jwksURI == "" && defaultExternalAddress == "" {
		return nil, fmt.Errorf("either the JWKS URI or the default external address, or both, must be set")
	}
	if len(pubKeys) == 0 {
		return nil, fmt.Errorf("no keys provided for validating keyset")
	}

	// Ensure the issuer URL meets the OIDC spec (this is the additional
	// validation the doc comment warns about).
	// https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderMetadata
	iss, err := url.Parse(issuerURL)
	if err != nil {
		return nil, err
	}
	if iss.Scheme != "https" {
		return nil, fmt.Errorf("issuer URL must use https scheme, got: %s", issuerURL)
	}
	if iss.RawQuery != "" {
		return nil, fmt.Errorf("issuer URL may not include a query, got: %s", issuerURL)
	}
	if iss.Fragment != "" {
		return nil, fmt.Errorf("issuer URL may not include a fragment, got: %s", issuerURL)
	}

	// Either use the provided JWKS URI or default to ExternalAddress plus
	// the JWKS path.
	if jwksURI == "" {
		const msg = "attempted to build jwks_uri from external " +
			"address %s, but could not construct a valid URL. Error: %v"

		if defaultExternalAddress == "" {
			return nil, fmt.Errorf(msg, defaultExternalAddress,
				fmt.Errorf("empty address"))
		}

		u := &url.URL{
			Scheme: "https",
			Host:   defaultExternalAddress,
			Path:   JWKSPath,
		}
		jwksURI = u.String()

		// TODO(mtaufen): I think we can probably expect ExternalAddress is
		// at most just host + port and skip the sanity check, but want to be
		// careful until that is confirmed.

		// Sanity check that the jwksURI we produced is the valid URL we expect.
		// This is just in case ExternalAddress came in as something weird,
		// like a scheme + host + port, instead of just host + port.
		parsed, err := url.Parse(jwksURI)
		if err != nil {
			return nil, fmt.Errorf(msg, defaultExternalAddress, err)
		} else if u.Scheme != parsed.Scheme ||
			u.Host != parsed.Host ||
			u.Path != parsed.Path {
			return nil, fmt.Errorf(msg, defaultExternalAddress,
				fmt.Errorf("got %v, expected %v", parsed, u))
		}
	} else {
		// Double-check that jwksURI is an https URL
		if u, err := url.Parse(jwksURI); err != nil {
			return nil, err
		} else if u.Scheme != "https" {
			return nil, fmt.Errorf("jwksURI requires https scheme, parsed as: %v", u.String())
		}
	}

	configJSON, err := openIDConfigJSON(issuerURL, jwksURI, pubKeys)
	if err != nil {
		return nil, fmt.Errorf("could not marshal issuer discovery JSON, error: %v", err)
	}

	keysetJSON, err := openIDKeysetJSON(pubKeys)
	if err != nil {
		return nil, fmt.Errorf("could not marshal issuer keys JSON, error: %v", err)
	}

	return &OpenIDMetadata{
		ConfigJSON:       configJSON,
		PublicKeysetJSON: keysetJSON,
	}, nil
}


openIDConfigJSON

// openIDConfigJSON returns the JSON OIDC Discovery Doc for the service
// account issuer.
func openIDConfigJSON(iss, jwksURI string, keys []interface{}) ([]byte, error) {
	keyset, errs := publicJWKSFromKeys(keys)
	if errs != nil {
		return nil, errs
	}

	metadata := openIDMetadata{
		Issuer:        iss,
		JWKSURI:       jwksURI,
		ResponseTypes: []string{"id_token"}, // Kubernetes only produces ID tokens
		SubjectTypes:  []string{"public"},   // https://openid.net/specs/openid-connect-core-1_0.html#SubjectIDTypes
		SigningAlgs:   getAlgs(keyset),      // REQUIRED by OIDC
	}

	metadataJSON, err := json.Marshal(metadata)
	if err != nil {
		return nil, fmt.Errorf("failed to marshal service account issuer metadata: %v", err)
	}

	return metadataJSON, nil
}


openIDKeysetJSON

// openIDKeysetJSON returns the JSON Web Key Set for the service account
// issuer's keys.
func openIDKeysetJSON(keys []interface{}) ([]byte, error) {
	keyset, errs := publicJWKSFromKeys(keys)
	if errs != nil {
		return nil, errs
	}

	keysetJSON, err := json.Marshal(keyset)
	if err != nil {
		return nil, fmt.Errorf("failed to marshal service account issuer JWKS: %v", err)
	}

	return keysetJSON, nil
}


getAlgs

func getAlgs(keys *jose.JSONWebKeySet) []string {
	algs := sets.NewString()
	for _, k := range keys.Keys {
		algs.Insert(k.Algorithm)
	}
	// Note: List returns a sorted slice.
	return algs.List()
}


publicJWKSFromKeys

// publicJWKSFromKeys constructs a JSONWebKeySet from a list of keys. The key
// set will only contain the public keys associated with the input keys.
func publicJWKSFromKeys(in []interface{}) (*jose.JSONWebKeySet, errors.Aggregate) {
	// Decode keys into a JWKS.
	var keys jose.JSONWebKeySet
	var errs []error
	for i, key := range in {
		var pubkey *jose.JSONWebKey
		var err error

		switch k := key.(type) {
		case publicKeyGetter:
			// This is a private key. Get its public key
			pubkey, err = jwkFromPublicKey(k.Public())
		default:
			pubkey, err = jwkFromPublicKey(k)
		}
		if err != nil {
			errs = append(errs, fmt.Errorf("error constructing JWK for key #%d: %v", i, err))
			continue
		}

		if !pubkey.Valid() {
			errs = append(errs, fmt.Errorf("key #%d not valid", i))
			continue
		}
		keys.Keys = append(keys.Keys, *pubkey)
	}
	if len(errs) != 0 {
		return nil, errors.NewAggregate(errs)
	}
	return &keys, nil
}


jwkFromPublicKey

func jwkFromPublicKey(publicKey crypto.PublicKey) (*jose.JSONWebKey, error) {
	alg, err := algorithmFromPublicKey(publicKey)
	if err != nil {
		return nil, err
	}

	keyID, err := keyIDFromPublicKey(publicKey)
	if err != nil {
		return nil, err
	}

	jwk := &jose.JSONWebKey{
		Algorithm: string(alg),
		Key:       publicKey,
		KeyID:     keyID,
		Use:       "sig",
	}

	if !jwk.IsPublic() {
		return nil, fmt.Errorf("JWK was not a public key! JWK: %v", jwk)
	}

	return jwk, nil
}


algorithmFromPublicKey

func algorithmFromPublicKey(publicKey crypto.PublicKey) (jose.SignatureAlgorithm, error) {
	switch pk := publicKey.(type) {
	case *rsa.PublicKey:
		// IMPORTANT: If this function is updated to support additional key sizes,
		// signerFromRSAPrivateKey in serviceaccount/jwt.go must also be
		// updated to support the same key sizes. Today we only support RS256.
		return jose.RS256, nil
	case *ecdsa.PublicKey:
		switch pk.Curve {
		case elliptic.P256():
			return jose.ES256, nil
		case elliptic.P384():
			return jose.ES384, nil
		case elliptic.P521():
			return jose.ES512, nil
		default:
			return "", fmt.Errorf("unknown private key curve, must be 256, 384, or 521")
		}
	case jose.OpaqueSigner:
		return jose.SignatureAlgorithm(pk.Public().Algorithm), nil
	default:
		return "", fmt.Errorf("unknown public key type, must be *rsa.PublicKey, *ecdsa.PublicKey, or jose.OpaqueSigner")
	}
}


TryAccept

func (nullLimiter) TryAccept() bool {
	return true
}


C

func (rt *realTimer) C() <-chan time.Time {
	return rt.timer.C
}


Reset

func (rt *realTimer) Reset(d time.Duration) bool {
	rt.next = time.Now().Add(d)
	return rt.timer.Reset(d)
}


Stop

func (rt *realTimer) Stop() bool {
	return rt.timer.Stop()
}


Now

func (rt *realTimer) Now() time.Time {
	return time.Now()
}


Remaining

func (rt *realTimer) Remaining() time.Duration {
	return rt.next.Sub(time.Now())
}


Since

func (rt *realTimer) Since(t time.Time) time.Duration {
	return time.Since(t)
}


Sleep

func (rt *realTimer) Sleep(d time.Duration) {
	time.Sleep(d)
}


NewBoundedFrequencyRunner

// NewBoundedFrequencyRunner creates a new BoundedFrequencyRunner instance,
// which will manage runs of the specified function.
//
// All runs will be async to the caller of BoundedFrequencyRunner.Run, but
// multiple runs are serialized. If the function needs to hold locks, it must
// take them internally.
//
// Runs of the function will have at least minInterval between them (from
// completion to next start), except that up to bursts may be allowed.  Burst
// runs are "accumulated" over time, one per minInterval up to burstRuns total.
// This can be used, for example, to mitigate the impact of expensive operations
// being called in response to user-initiated operations. Run requests that
// would violate the minInterval are coalesced and run at the next opportunity.
//
// The function will be run at least once per maxInterval. For example, this can
// force periodic refreshes of state in the absence of anyone calling Run.
//
// Examples:
//
// NewBoundedFrequencyRunner("name", fn, time.Second, 5*time.Second, 1)
// - fn will have at least 1 second between runs
// - fn will have no more than 5 seconds between runs
//
// NewBoundedFrequencyRunner("name", fn, 3*time.Second, 10*time.Second, 3)
// - fn will have at least 3 seconds between runs, with up to 3 burst runs
// - fn will have no more than 10 seconds between runs
//
// The maxInterval must be greater than or equal to the minInterval,  If the
// caller passes a maxInterval less than minInterval, this function will panic.
func NewBoundedFrequencyRunner(name string, fn func(), minInterval, maxInterval time.Duration, burstRuns int) *BoundedFrequencyRunner {
	timer := &realTimer{timer: time.NewTimer(0)} // will tick immediately
	<-timer.C()                                  // consume the first tick
	return construct(name, fn, minInterval, maxInterval, burstRuns, timer)
}


construct

// Make an instance with dependencies injected.
func construct(name string, fn func(), minInterval, maxInterval time.Duration, burstRuns int, timer timer) *BoundedFrequencyRunner {
	if maxInterval < minInterval {
		panic(fmt.Sprintf("%s: maxInterval (%v) must be >= minInterval (%v)", name, maxInterval, minInterval))
	}
	if timer == nil {
		panic(fmt.Sprintf("%s: timer must be non-nil", name))
	}

	bfr := &BoundedFrequencyRunner{
		name:        name,
		fn:          fn,
		minInterval: minInterval,
		maxInterval: maxInterval,
		run:         make(chan struct{}, 1),
		retry:       make(chan struct{}, 1),
		timer:       timer,
	}
	if minInterval == 0 {
		bfr.limiter = nullLimiter{}
	} else {
		// allow burst updates in short succession
		qps := float32(time.Second) / float32(minInterval)
		bfr.limiter = flowcontrol.NewTokenBucketRateLimiterWithClock(qps, burstRuns, timer)
	}
	return bfr
}


Loop

// Loop handles the periodic timer and run requests.  This is expected to be
// called as a goroutine.
func (bfr *BoundedFrequencyRunner) Loop(stop <-chan struct{}) {
	klog.V(3).Infof("%s Loop running", bfr.name)
	bfr.timer.Reset(bfr.maxInterval)
	for {
		select {
		case <-stop:
			bfr.stop()
			klog.V(3).Infof("%s Loop stopping", bfr.name)
			return
		case <-bfr.timer.C():
			bfr.tryRun()
		case <-bfr.run:
			bfr.tryRun()
		case <-bfr.retry:
			bfr.doRetry()
		}
	}
}


Run

// Run the function as soon as possible.  If this is called while Loop is not
// running, the call may be deferred indefinitely.
// If there is already a queued request to call the underlying function, it
// may be dropped - it is just guaranteed that we will try calling the
// underlying function as soon as possible starting from now.
func (bfr *BoundedFrequencyRunner) Run() {
	// If it takes a lot of time to run the underlying function, noone is really
	// processing elements from <run> channel. So to avoid blocking here on the
	// putting element to it, we simply skip it if there is already an element
	// in it.
	select {
	case bfr.run <- struct{}{}:
	default:
	}
}


RetryAfter

// RetryAfter ensures that the function will run again after no later than interval. This
// can be called from inside a run of the BoundedFrequencyRunner's function, or
// asynchronously.
func (bfr *BoundedFrequencyRunner) RetryAfter(interval time.Duration) {
	// This could be called either with or without bfr.mu held, so we can't grab that
	// lock, and therefore we can't update the timer directly.

	// If the Loop thread is currently running fn then it may be a while before it
	// processes our retry request. But we want to retry at interval from now, not at
	// interval from "whenever doRetry eventually gets called". So we convert to
	// absolute time.
	retryTime := bfr.timer.Now().Add(interval)

	// We can't just write retryTime to a channel because there could be multiple
	// RetryAfter calls before Loop gets a chance to read from the channel. So we
	// record the soonest requested retry time in bfr.retryTime and then only signal
	// the Loop thread once, just like Run does.
	bfr.retryMu.Lock()
	defer bfr.retryMu.Unlock()
	if !bfr.retryTime.IsZero() && bfr.retryTime.Before(retryTime) {
		return
	}
	bfr.retryTime = retryTime

	select {
	case bfr.retry <- struct{}{}:
	default:
	}
}


stop

// assumes the lock is not held
func (bfr *BoundedFrequencyRunner) stop() {
	bfr.mu.Lock()
	defer bfr.mu.Unlock()
	bfr.limiter.Stop()
	bfr.timer.Stop()
}


doRetry

// assumes the lock is not held
func (bfr *BoundedFrequencyRunner) doRetry() {
	bfr.mu.Lock()
	defer bfr.mu.Unlock()
	bfr.retryMu.Lock()
	defer bfr.retryMu.Unlock()

	if bfr.retryTime.IsZero() {
		return
	}

	// Timer wants an interval not an absolute time, so convert retryTime back now
	retryInterval := bfr.retryTime.Sub(bfr.timer.Now())
	bfr.retryTime = time.Time{}
	if retryInterval < bfr.timer.Remaining() {
		klog.V(3).Infof("%s: retrying in %v", bfr.name, retryInterval)
		bfr.timer.Stop()
		bfr.timer.Reset(retryInterval)
	}
}


tryRun

// assumes the lock is not held
func (bfr *BoundedFrequencyRunner) tryRun() {
	bfr.mu.Lock()
	defer bfr.mu.Unlock()

	if bfr.limiter.TryAccept() {
		// We're allowed to run the function right now.
		bfr.fn()
		bfr.lastRun = bfr.timer.Now()
		bfr.timer.Stop()
		bfr.timer.Reset(bfr.maxInterval)
		klog.V(3).Infof("%s: ran, next possible in %v, periodic in %v", bfr.name, bfr.minInterval, bfr.maxInterval)
		return
	}

	// It can't run right now, figure out when it can run next.
	elapsed := bfr.timer.Since(bfr.lastRun)   // how long since last run
	nextPossible := bfr.minInterval - elapsed // time to next possible run
	nextScheduled := bfr.timer.Remaining()    // time to next scheduled run
	klog.V(4).Infof("%s: %v since last run, possible in %v, scheduled in %v", bfr.name, elapsed, nextPossible, nextScheduled)

	// It's hard to avoid race conditions in the unit tests unless we always reset
	// the timer here, even when it's unchanged
	if nextPossible < nextScheduled {
		nextScheduled = nextPossible
	}
	bfr.timer.Stop()
	bfr.timer.Reset(nextScheduled)
}


NewRunner

// NewRunner makes a runner for the given function(s). The function(s) should loop until
// the channel is closed.
func NewRunner(f ...func(stop chan struct{})) *Runner {
	return &Runner{loopFuncs: f}
}


Start

// Start begins running.
func (r *Runner) Start() {
	r.lock.Lock()
	defer r.lock.Unlock()
	if r.stop == nil {
		c := make(chan struct{})
		r.stop = &c
		for i := range r.loopFuncs {
			go r.loopFuncs[i](*r.stop)
		}
	}
}


Stop

// Stop stops running.
func (r *Runner) Stop() {
	r.lock.Lock()
	defer r.lock.Unlock()
	if r.stop != nil {
		close(*r.stop)
		r.stop = nil
	}
}


Merge

func (f MergeFunc) Merge(source string, update interface{}) error {
	return f(source, update)
}


NewMux

// NewMux creates a new mux that can merge changes from multiple sources.
func NewMux(merger Merger) *Mux {
	mux := &Mux{
		sources: make(map[string]chan interface{}),
		merger:  merger,
	}
	return mux
}


ChannelWithContext

// ChannelWithContext returns a channel where a configuration source
// can send updates of new configurations. Multiple calls with the same
// source will return the same channel. This allows change and state based sources
// to use the same channel. Different source names however will be treated as a
// union.
func (m *Mux) ChannelWithContext(ctx context.Context, source string) chan interface{} {
	if len(source) == 0 {
		panic("Channel given an empty name")
	}
	m.sourceLock.Lock()
	defer m.sourceLock.Unlock()
	channel, exists := m.sources[source]
	if exists {
		return channel
	}
	newChannel := make(chan interface{})
	m.sources[source] = newChannel

	go wait.Until(func() { m.listen(source, newChannel) }, 0, ctx.Done())
	return newChannel
}


listen

func (m *Mux) listen(source string, listenChannel <-chan interface{}) {
	for update := range listenChannel {
		m.merger.Merge(source, update)
	}
}


MergedState

func (f AccessorFunc) MergedState() interface{} {
	return f()
}


OnUpdate

func (f ListenerFunc) OnUpdate(instance interface{}) {
	f(instance)
}


NewBroadcaster

// NewBroadcaster registers a set of listeners that support the Listener interface
// and notifies them all on changes.
func NewBroadcaster() *Broadcaster {
	return &Broadcaster{}
}


Add

// Add registers listener to receive updates of changes.
func (b *Broadcaster) Add(listener Listener) {
	b.listenerLock.Lock()
	defer b.listenerLock.Unlock()
	b.listeners = append(b.listeners, listener)
}


Notify

// Notify notifies all listeners.
func (b *Broadcaster) Notify(instance interface{}) {
	b.listenerLock.RLock()
	listeners := b.listeners
	b.listenerLock.RUnlock()
	for _, listener := range listeners {
		listener.OnUpdate(instance)
	}
}


protoStr

func protoStr(proto v1.Protocol) string {
	return strings.ToLower(string(proto))
}


parametersWithFamily

func parametersWithFamily(isIPv6 bool, parameters ...string) []string {
	if isIPv6 {
		parameters = append(parameters, "-f", "ipv6")
	}
	return parameters
}


ClearEntriesForIP

// ClearEntriesForIP uses the conntrack tool to delete the conntrack entries
// for the UDP connections specified by the given service IP
func ClearEntriesForIP(execer exec.Interface, ip string, protocol v1.Protocol) error {
	parameters := parametersWithFamily(utilnet.IsIPv6String(ip), "-D", "--orig-dst", ip, "-p", protoStr(protocol))
	err := Exec(execer, parameters...)
	if err != nil && !strings.Contains(err.Error(), NoConnectionToDelete) {
		// TODO: Better handling for deletion failure. When failure occur, stale udp connection may not get flushed.
		// These stale udp connection will keep black hole traffic. Making this a best effort operation for now, since it
		// is expensive to baby-sit all udp connections to kubernetes services.
		return fmt.Errorf("error deleting connection tracking state for UDP service IP: %s, error: %v", ip, err)
	}
	return nil
}


Exec

// Exec executes the conntrack tool using the given parameters
func Exec(execer exec.Interface, parameters ...string) error {
	conntrackPath, err := execer.LookPath("conntrack")
	if err != nil {
		return fmt.Errorf("error looking for path of conntrack: %v", err)
	}
	klog.V(4).Infof("Clearing conntrack entries %v", parameters)
	output, err := execer.Command(conntrackPath, parameters...).CombinedOutput()
	if err != nil {
		return fmt.Errorf("conntrack command returned: %q, error message: %s", string(output), err)
	}
	klog.V(4).Infof("Conntrack entries deleted %s", string(output))
	return nil
}


ClearEntriesForPort

// ClearEntriesForPort uses the conntrack tool to delete the conntrack entries
// for connections specified by the port.
// When a packet arrives, it will not go through NAT table again, because it is not "the first" packet.
// The solution is clearing the conntrack. Known issues:
// https://github.com/docker/docker/issues/8795
// https://github.com/kubernetes/kubernetes/issues/31983
func ClearEntriesForPort(execer exec.Interface, port int, isIPv6 bool, protocol v1.Protocol) error {
	if port <= 0 {
		return fmt.Errorf("wrong port number. The port number must be greater than zero")
	}
	parameters := parametersWithFamily(isIPv6, "-D", "-p", protoStr(protocol), "--dport", strconv.Itoa(port))
	err := Exec(execer, parameters...)
	if err != nil && !strings.Contains(err.Error(), NoConnectionToDelete) {
		return fmt.Errorf("error deleting conntrack entries for UDP port: %d, error: %v", port, err)
	}
	return nil
}


ClearEntriesForNAT

// ClearEntriesForNAT uses the conntrack tool to delete the conntrack entries
// for connections specified by the {origin, dest} IP pair.
func ClearEntriesForNAT(execer exec.Interface, origin, dest string, protocol v1.Protocol) error {
	parameters := parametersWithFamily(utilnet.IsIPv6String(origin), "-D", "--orig-dst", origin, "--dst-nat", dest,
		"-p", protoStr(protocol))
	err := Exec(execer, parameters...)
	if err != nil && !strings.Contains(err.Error(), NoConnectionToDelete) {
		// TODO: Better handling for deletion failure. When failure occur, stale udp connection may not get flushed.
		// These stale udp connection will keep black hole traffic. Making this a best effort operation for now, since it
		// is expensive to baby sit all udp connections to kubernetes services.
		return fmt.Errorf("error deleting conntrack entries for UDP peer {%s, %s}, error: %v", origin, dest, err)
	}
	return nil
}


ClearEntriesForPortNAT

// ClearEntriesForPortNAT uses the conntrack tool to delete the conntrack entries
// for connections specified by the {dest IP, port} pair.
// Known issue:
// https://github.com/kubernetes/kubernetes/issues/59368
func ClearEntriesForPortNAT(execer exec.Interface, dest string, port int, protocol v1.Protocol) error {
	if port <= 0 {
		return fmt.Errorf("wrong port number. The port number must be greater than zero")
	}
	parameters := parametersWithFamily(utilnet.IsIPv6String(dest), "-D", "-p", protoStr(protocol), "--dport", strconv.Itoa(port), "--dst-nat", dest)
	err := Exec(execer, parameters...)
	if err != nil && !strings.Contains(err.Error(), NoConnectionToDelete) {
		return fmt.Errorf("error deleting conntrack entries for UDP port: %d, error: %v", port, err)
	}
	return nil
}


tempCoveragePath

// tempCoveragePath returns a temporary file to write coverage information to.
// The file is in the same directory as the destination, ensuring os.Rename will work.
func tempCoveragePath() string {
	return coverageFile + ".tmp"
}


InitCoverage

// InitCoverage is called from the dummy unit test to prepare Go's coverage framework.
// Clients should never need to call it.
func InitCoverage(name string) {
	// We read the coverage destination in from the KUBE_COVERAGE_FILE env var,
	// or if it's empty we just use a default in /tmp
	coverageFile = os.Getenv("KUBE_COVERAGE_FILE")
	if coverageFile == "" {
		coverageFile = "/tmp/k8s-" + name + ".cov"
	}
	fmt.Println("Dumping coverage information to " + coverageFile)

	flushInterval := 5 * time.Second
	requestedInterval := os.Getenv("KUBE_COVERAGE_FLUSH_INTERVAL")
	if requestedInterval != "" {
		if duration, err := time.ParseDuration(requestedInterval); err == nil {
			flushInterval = duration
		} else {
			panic("Invalid KUBE_COVERAGE_FLUSH_INTERVAL value; try something like '30s'.")
		}
	}

	// Set up the unit test framework with the required arguments to activate test coverage.
	flag.CommandLine.Parse([]string{"-test.coverprofile", tempCoveragePath()})

	// Begin periodic logging
	go wait.Forever(FlushCoverage, flushInterval)
}


FlushCoverage

// FlushCoverage flushes collected coverage information to disk.
// The destination file is configured at startup and cannot be changed.
// Calling this function also sends a line like "coverage: 5% of statements" to stdout.
func FlushCoverage() {
	// We're not actually going to run any tests, but we need Go to think we did so it writes
	// coverage information to disk. To achieve this, we create a bunch of empty test suites and
	// have it "run" them.
	tests := []testing.InternalTest{}
	benchmarks := []testing.InternalBenchmark{}
	examples := []testing.InternalExample{}
	fuzztargets := []testing.InternalFuzzTarget{}

	var deps fakeTestDeps

	dummyRun := testing.MainStart(deps, tests, benchmarks, fuzztargets, examples)
	dummyRun.Run()

	// Once it writes to the temporary path, we move it to the intended path.
	// This gets us atomic updates from the perspective of another process trying to access
	// the file.
	if err := os.Rename(tempCoveragePath(), coverageFile); err != nil {
		klog.Errorf("Couldn't move coverage file from %s to %s", coverageFile, tempCoveragePath())
	}
}


InitCoverage

// InitCoverage is illegal when not running with coverage.
func InitCoverage(name string) {
	panic("Called InitCoverage when not built with coverage instrumentation.")
}


FlushCoverage

// FlushCoverage is a no-op when not running with coverage.
func FlushCoverage() {

}


GetEnvAsStringOrFallback

// GetEnvAsStringOrFallback returns the env variable for the given key
// and falls back to the given defaultValue if not set
func GetEnvAsStringOrFallback(key, defaultValue string) string {
	if v := os.Getenv(key); v != "" {
		return v
	}
	return defaultValue
}


GetEnvAsIntOrFallback

// GetEnvAsIntOrFallback returns the env variable (parsed as integer) for
// the given key and falls back to the given defaultValue if not set
func GetEnvAsIntOrFallback(key string, defaultValue int) (int, error) {
	if v := os.Getenv(key); v != "" {
		value, err := strconv.Atoi(v)
		if err != nil {
			return defaultValue, err
		}
		return value, nil
	}
	return defaultValue, nil
}


// GetEnvAsFloat64OrFallback returns the env variable (parsed as float64) for
// the given key and falls back to the given defaultValue if not set
func GetEnvAsFloat64OrFallback(key string, defaultValue float64) (float64, error) {
	if v := os.Getenv(key); v != "" {
		value, err := strconv.ParseFloat(v, 64)
		if err != nil {
			return defaultValue, err
		}
		return value, nil
	}
	return defaultValue, nil
}


NewTempFs

// NewTempFs returns a fake Filesystem in temporary directory, useful for unit tests
func NewTempFs() Filesystem {
	path, _ := os.MkdirTemp("", "tmpfs")
	return &DefaultFs{
		root: path,
	}
}


prefix

func (fs *DefaultFs) prefix(path string) string {
	if len(fs.root) == 0 {
		return path
	}
	return filepath.Join(fs.root, path)
}


Stat

// Stat via os.Stat
func (fs *DefaultFs) Stat(name string) (os.FileInfo, error) {
	return os.Stat(fs.prefix(name))
}


Create

// Create via os.Create
func (fs *DefaultFs) Create(name string) (File, error) {
	file, err := os.Create(fs.prefix(name))
	if err != nil {
		return nil, err
	}
	return &defaultFile{file}, nil
}


Rename

// Rename via os.Rename
func (fs *DefaultFs) Rename(oldpath, newpath string) error {
	if !strings.HasPrefix(oldpath, fs.root) {
		oldpath = fs.prefix(oldpath)
	}
	if !strings.HasPrefix(newpath, fs.root) {
		newpath = fs.prefix(newpath)
	}
	return os.Rename(oldpath, newpath)
}


MkdirAll

// MkdirAll via os.MkdirAll
func (fs *DefaultFs) MkdirAll(path string, perm os.FileMode) error {
	return os.MkdirAll(fs.prefix(path), perm)
}


Chtimes

// Chtimes via os.Chtimes
func (fs *DefaultFs) Chtimes(name string, atime time.Time, mtime time.Time) error {
	return os.Chtimes(fs.prefix(name), atime, mtime)
}


RemoveAll

// RemoveAll via os.RemoveAll
func (fs *DefaultFs) RemoveAll(path string) error {
	return os.RemoveAll(fs.prefix(path))
}


Remove

// Remove via os.RemoveAll
func (fs *DefaultFs) Remove(name string) error {
	return os.Remove(fs.prefix(name))
}


ReadFile

// ReadFile via os.ReadFile
func (fs *DefaultFs) ReadFile(filename string) ([]byte, error) {
	return os.ReadFile(fs.prefix(filename))
}


TempDir

// TempDir via os.MkdirTemp
func (fs *DefaultFs) TempDir(dir, prefix string) (string, error) {
	return os.MkdirTemp(fs.prefix(dir), prefix)
}


TempFile

// TempFile via os.CreateTemp
func (fs *DefaultFs) TempFile(dir, prefix string) (File, error) {
	file, err := os.CreateTemp(fs.prefix(dir), prefix)
	if err != nil {
		return nil, err
	}
	return &defaultFile{file}, nil
}


ReadDir

// ReadDir via os.ReadDir
func (fs *DefaultFs) ReadDir(dirname string) ([]os.DirEntry, error) {
	return os.ReadDir(fs.prefix(dirname))
}


Walk

// Walk via filepath.Walk
func (fs *DefaultFs) Walk(root string, walkFn filepath.WalkFunc) error {
	return filepath.Walk(fs.prefix(root), walkFn)
}


Name

// Name via os.File.Name
func (file *defaultFile) Name() string {
	return file.file.Name()
}


Write

// Write via os.File.Write
func (file *defaultFile) Write(b []byte) (n int, err error) {
	return file.file.Write(b)
}


Sync

// Sync via os.File.Sync
func (file *defaultFile) Sync() error {
	return file.file.Sync()
}


Close

// Close via os.File.Close
func (file *defaultFile) Close() error {
	return file.file.Close()
}


NewFsnotifyWatcher

// NewFsnotifyWatcher returns an implementation of FSWatcher that continuously listens for
// fsnotify events and calls the event handler as soon as an event is received.
func NewFsnotifyWatcher() FSWatcher {
	return &fsnotifyWatcher{}
}


AddWatch

func (w *fsnotifyWatcher) AddWatch(path string) error {
	return w.watcher.Add(path)
}


Init

func (w *fsnotifyWatcher) Init(eventHandler FSEventHandler, errorHandler FSErrorHandler) error {
	var err error
	w.watcher, err = fsnotify.NewWatcher()
	if err != nil {
		return err
	}

	w.eventHandler = eventHandler
	w.errorHandler = errorHandler
	return nil
}


Run

func (w *fsnotifyWatcher) Run() {
	go func() {
		defer w.watcher.Close()
		for {
			select {
			case event := <-w.watcher.Events:
				if w.eventHandler != nil {
					w.eventHandler(event)
				}
			case err := <-w.watcher.Errors:
				if w.errorHandler != nil {
					w.errorHandler(err)
				}
			}
		}
	}()
}


Set

// Set sets the flag value
func (v *IPVar) Set(s string) error {
	if len(s) == 0 {
		v.Val = nil
		return nil
	}
	if netutils.ParseIPSloppy(s) == nil {
		return fmt.Errorf("%q is not a valid IP address", s)
	}
	if v.Val == nil {
		// it's okay to panic here since this is programmer error
		panic("the string pointer passed into IPVar should not be nil")
	}
	*v.Val = s
	return nil
}


String

// String returns the flag value
func (v *IPVar) String() string {
	if v.Val == nil {
		return ""
	}
	return *v.Val
}


Type

// Type gets the flag type
func (v *IPVar) Type() string {
	return "ip"
}


Set

// Set sets the flag value
func (v *IPPortVar) Set(s string) error {
	if len(s) == 0 {
		v.Val = nil
		return nil
	}

	if v.Val == nil {
		// it's okay to panic here since this is programmer error
		panic("the string pointer passed into IPPortVar should not be nil")
	}

	// Both IP and IP:port are valid.
	// Attempt to parse into IP first.
	if netutils.ParseIPSloppy(s) != nil {
		*v.Val = s
		return nil
	}

	// Can not parse into IP, now assume IP:port.
	host, port, err := net.SplitHostPort(s)
	if err != nil {
		return fmt.Errorf("%q is not in a valid format (ip or ip:port): %v", s, err)
	}
	if netutils.ParseIPSloppy(host) == nil {
		return fmt.Errorf("%q is not a valid IP address", host)
	}
	if _, err := netutils.ParsePort(port, true); err != nil {
		return fmt.Errorf("%q is not a valid number", port)
	}
	*v.Val = s
	return nil
}


String

// String returns the flag value
func (v *IPPortVar) String() string {
	if v.Val == nil {
		return ""
	}
	return *v.Val
}


Type

// Type gets the flag type
func (v *IPPortVar) Type() string {
	return "ipport"
}


Set

// Set sets the flag value
func (v PortRangeVar) Set(s string) error {
	if _, err := utilnet.ParsePortRange(s); err != nil {
		return fmt.Errorf("%q is not a valid port range: %v", s, err)
	}
	if v.Val == nil {
		// it's okay to panic here since this is programmer error
		panic("the string pointer passed into PortRangeVar should not be nil")
	}
	*v.Val = s
	return nil
}


String

// String returns the flag value
func (v PortRangeVar) String() string {
	if v.Val == nil {
		return ""
	}
	return *v.Val
}


Type

// Type gets the flag type
func (v PortRangeVar) Type() string {
	return "port-range"
}


Set

// Set sets the flag value
func (v *ReservedMemoryVar) Set(s string) error {
	if v.Value == nil {
		return fmt.Errorf("no target (nil pointer to *[]MemoryReservation")
	}

	if s == "" {
		v.Value = nil
		return nil
	}

	if !v.initialized || *v.Value == nil {
		*v.Value = make([]kubeletconfig.MemoryReservation, 0)
		v.initialized = true
	}

	if s == "" {
		return nil
	}

	numaNodeReservations := strings.Split(s, ";")
	for _, reservation := range numaNodeReservations {
		numaNodeReservation := strings.Split(reservation, ":")
		if len(numaNodeReservation) != 2 {
			return fmt.Errorf("the reserved memory has incorrect format, expected numaNodeID:type=quantity[,type=quantity...], got %s", reservation)
		}
		memoryTypeReservations := strings.Split(numaNodeReservation[1], ",")
		if len(memoryTypeReservations) < 1 {
			return fmt.Errorf("the reserved memory has incorrect format, expected numaNodeID:type=quantity[,type=quantity...], got %s", reservation)
		}
		numaNodeID, err := strconv.Atoi(numaNodeReservation[0])
		if err != nil {
			return fmt.Errorf("failed to convert the NUMA node ID, exptected integer, got %s", numaNodeReservation[0])
		}

		memoryReservation := kubeletconfig.MemoryReservation{
			NumaNode: int32(numaNodeID),
			Limits:   map[v1.ResourceName]resource.Quantity{},
		}

		for _, memoryTypeReservation := range memoryTypeReservations {
			limit := strings.Split(memoryTypeReservation, "=")
			if len(limit) != 2 {
				return fmt.Errorf("the reserved limit has incorrect value, expected type=quantatity, got %s", memoryTypeReservation)
			}

			resourceName := v1.ResourceName(limit[0])
			if resourceName != v1.ResourceMemory && !corev1helper.IsHugePageResourceName(resourceName) {
				return fmt.Errorf("memory type conversion error, unknown type: %q", resourceName)
			}

			q, err := resource.ParseQuantity(limit[1])
			if err != nil {
				return fmt.Errorf("failed to parse the quantatity, expected quantatity, got %s", limit[1])
			}

			memoryReservation.Limits[v1.ResourceName(limit[0])] = q
		}
		*v.Value = append(*v.Value, memoryReservation)
	}
	return nil
}


String

// String returns the flag value
func (v *ReservedMemoryVar) String() string {
	if v == nil || v.Value == nil {
		return ""
	}

	var slices []string
	for _, reservedMemory := range *v.Value {
		var limits []string
		for resourceName, q := range reservedMemory.Limits {
			limits = append(limits, fmt.Sprintf("%s=%s", resourceName, q.String()))
		}

		sort.Strings(limits)
		slices = append(slices, fmt.Sprintf("%d:%s", reservedMemory.NumaNode, strings.Join(limits, ",")))
	}

	sort.Strings(slices)
	return strings.Join(slices, ",")
}


Type

// Type gets the flag type
func (v *ReservedMemoryVar) Type() string {
	return "reserved-memory"
}


Set

// Set sets the flag value
func (t RegisterWithTaintsVar) Set(s string) error {
	if len(s) == 0 {
		*t.Value = nil
		return nil
	}
	sts := strings.Split(s, ",")
	corev1Taints, _, err := utiltaints.ParseTaints(sts)
	if err != nil {
		return err
	}
	var taints []v1.Taint
	for _, ct := range corev1Taints {
		taints = append(taints, v1.Taint{Key: ct.Key, Value: ct.Value, Effect: ct.Effect})
	}
	*t.Value = taints
	return nil
}


String

// String returns the flag value
func (t RegisterWithTaintsVar) String() string {
	if len(*t.Value) == 0 {
		return ""
	}
	var taints []string
	for _, taint := range *t.Value {
		taints = append(taints, fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect))
	}
	return strings.Join(taints, ",")
}


Type

// Type gets the flag type
func (t RegisterWithTaintsVar) Type() string {
	return "[]v1.Taint"
}


Acquire

// Acquire is not implemented on non-unix systems.
func Acquire(path string) error {
	return nil
}


// Acquire acquires a lock on a file for the duration of the process. This method
// is reentrant.
func Acquire(path string) error {
	fd, err := unix.Open(path, unix.O_CREAT|unix.O_RDWR|unix.O_CLOEXEC, 0600)
	if err != nil {
		return err
	}

	// We don't need to close the fd since we should hold
	// it until the process exits.

	return unix.Flock(fd, unix.LOCK_EX)
}


SafeToRetry

// SafeToRetry returns an error if the durationBeforeRetry period for the given
// lastErrorTime has not yet expired. Otherwise it returns nil.
func (expBackoff *ExponentialBackoff) SafeToRetry(operationName string) error {
	if time.Since(expBackoff.lastErrorTime) <= expBackoff.durationBeforeRetry {
		return NewExponentialBackoffError(operationName, *expBackoff)
	}

	return nil
}


Update

func (expBackoff *ExponentialBackoff) Update(err *error) {
	if expBackoff.durationBeforeRetry == 0 {
		expBackoff.durationBeforeRetry = initialDurationBeforeRetry
	} else {
		expBackoff.durationBeforeRetry = 2 * expBackoff.durationBeforeRetry
		if expBackoff.durationBeforeRetry > maxDurationBeforeRetry {
			expBackoff.durationBeforeRetry = maxDurationBeforeRetry
		}
	}

	expBackoff.lastError = *err
	expBackoff.lastErrorTime = time.Now()
}


GenerateNoRetriesPermittedMsg

func (expBackoff *ExponentialBackoff) GenerateNoRetriesPermittedMsg(operationName string) string {
	return fmt.Sprintf("Operation for %q failed. No retries permitted until %v (durationBeforeRetry %v). Error: %v",
		operationName,
		expBackoff.lastErrorTime.Add(expBackoff.durationBeforeRetry),
		expBackoff.durationBeforeRetry,
		expBackoff.lastError)
}


NewExponentialBackoffError

// NewExponentialBackoffError returns a new instance of ExponentialBackoff error.
func NewExponentialBackoffError(
	operationName string, expBackoff ExponentialBackoff) error {
	return exponentialBackoffError{
		operationName: operationName,
		expBackoff:    expBackoff,
	}
}


IsExponentialBackoff

// IsExponentialBackoff returns true if an error returned from GoroutineMap
// indicates that a new operation can not be started because
// exponentialBackOffOnError is enabled and a previous operation with the same
// operation failed within the durationBeforeRetry period.
func IsExponentialBackoff(err error) bool {
	switch err.(type) {
	case exponentialBackoffError:
		return true
	default:
		return false
	}
}


Error

func (err exponentialBackoffError) Error() string {
	return fmt.Sprintf(
		"Failed to create operation with name %q. An operation with that name failed at %v. No retries permitted until %v (%v). Last error: %q.",
		err.operationName,
		err.expBackoff.lastErrorTime,
		err.expBackoff.lastErrorTime.Add(err.expBackoff.durationBeforeRetry),
		err.expBackoff.durationBeforeRetry,
		err.expBackoff.lastError)
}


NewGoRoutineMap

// NewGoRoutineMap returns a new instance of GoRoutineMap.
func NewGoRoutineMap(exponentialBackOffOnError bool) GoRoutineMap {
	g := &goRoutineMap{
		operations:                make(map[string]operation),
		exponentialBackOffOnError: exponentialBackOffOnError,
	}

	g.cond = sync.NewCond(&g.lock)
	return g
}


Run

func (grm *goRoutineMap) Run(
	operationName string,
	operationFunc func() error) error {
	grm.lock.Lock()
	defer grm.lock.Unlock()

	existingOp, exists := grm.operations[operationName]
	if exists {
		// Operation with name exists
		if existingOp.operationPending {
			return NewAlreadyExistsError(operationName)
		}

		if err := existingOp.expBackoff.SafeToRetry(operationName); err != nil {
			return err
		}
	}

	grm.operations[operationName] = operation{
		operationPending: true,
		expBackoff:       existingOp.expBackoff,
	}
	go func() (err error) {
		// Handle unhandled panics (very unlikely)
		defer k8sRuntime.HandleCrash()
		// Handle completion of and error, if any, from operationFunc()
		defer grm.operationComplete(operationName, &err)
		// Handle panic, if any, from operationFunc()
		defer k8sRuntime.RecoverFromPanic(&err)
		return operationFunc()
	}()

	return nil
}


operationComplete

// operationComplete handles the completion of a goroutine run in the
// goRoutineMap.
func (grm *goRoutineMap) operationComplete(
	operationName string, err *error) {
	// Defer operations are executed in Last-In is First-Out order. In this case
	// the lock is acquired first when operationCompletes begins, and is
	// released when the method finishes, after the lock is released cond is
	// signaled to wake waiting goroutine.
	defer grm.cond.Signal()
	grm.lock.Lock()
	defer grm.lock.Unlock()

	if *err == nil || !grm.exponentialBackOffOnError {
		// Operation completed without error, or exponentialBackOffOnError disabled
		delete(grm.operations, operationName)
		if *err != nil {
			// Log error
			klog.Errorf("operation for %q failed with: %v",
				operationName,
				*err)
		}
	} else {
		// Operation completed with error and exponentialBackOffOnError Enabled
		existingOp := grm.operations[operationName]
		existingOp.expBackoff.Update(err)
		existingOp.operationPending = false
		grm.operations[operationName] = existingOp

		// Log error
		klog.Errorf("%v",
			existingOp.expBackoff.GenerateNoRetriesPermittedMsg(operationName))
	}
}


IsOperationPending

func (grm *goRoutineMap) IsOperationPending(operationName string) bool {
	grm.lock.RLock()
	defer grm.lock.RUnlock()
	existingOp, exists := grm.operations[operationName]
	if exists && existingOp.operationPending {
		return true
	}
	return false
}


Wait

func (grm *goRoutineMap) Wait() {
	grm.lock.Lock()
	defer grm.lock.Unlock()

	for len(grm.operations) > 0 {
		grm.cond.Wait()
	}
}


WaitForCompletion

func (grm *goRoutineMap) WaitForCompletion() {
	grm.lock.Lock()
	defer grm.lock.Unlock()

	for {
		if len(grm.operations) == 0 || grm.nothingPending() {
			break
		} else {
			grm.cond.Wait()
		}
	}
}


nothingPending

// Check if any operation is pending. Already assumes caller has the
// necessary locks
func (grm *goRoutineMap) nothingPending() bool {
	nothingIsPending := true
	for _, operation := range grm.operations {
		if operation.operationPending {
			nothingIsPending = false
			break
		}
	}
	return nothingIsPending
}


NewAlreadyExistsError

// NewAlreadyExistsError returns a new instance of AlreadyExists error.
func NewAlreadyExistsError(operationName string) error {
	return alreadyExistsError{operationName}
}


IsAlreadyExists

// IsAlreadyExists returns true if an error returned from GoRoutineMap indicates
// a new operation can not be started because an operation with the same
// operation name is already executing.
func IsAlreadyExists(err error) bool {
	switch err.(type) {
	case alreadyExistsError:
		return true
	default:
		return false
	}
}


Error

func (err alreadyExistsError) Error() string {
	return fmt.Sprintf(
		"Failed to create operation with name %q. An operation with that name is already executing.",
		err.operationName)
}


DeepHashObject

// DeepHashObject writes specified object to hash using the spew library
// which follows pointers and prints actual values of the nested objects
// ensuring the hash does not change when a pointer changes.
func DeepHashObject(hasher hash.Hash, objectToWrite interface{}) {
	hasher.Reset()
	printer := spew.ConfigState{
		Indent:         " ",
		SortKeys:       true,
		DisableMethods: true,
		SpewKeys:       true,
	}
	printer.Fprintf(hasher, "%#v", objectToWrite)
}


New

// New creates a new handler that guarantees all notify functions are run after the critical
// section exits (or is interrupted by the OS), then invokes the final handler. If no final
// handler is specified, the default final is `os.Exit(1)`. A handler can only be used for
// one critical section.
func New(final func(os.Signal), notify ...func()) *Handler {
	return &Handler{
		final:  final,
		notify: notify,
	}
}


Close

// Close executes all the notification handlers if they have not yet been executed.
func (h *Handler) Close() {
	h.once.Do(func() {
		for _, fn := range h.notify {
			fn()
		}
	})
}


Signal

// Signal is called when an os.Signal is received, and guarantees that all notifications
// are executed, then the final handler is executed. This function should only be called once
// per Handler instance.
func (h *Handler) Signal(s os.Signal) {
	h.once.Do(func() {
		for _, fn := range h.notify {
			fn()
		}
		if h.final == nil {
			os.Exit(1)
		}
		h.final(s)
	})
}


Run

// Run ensures that any notifications are invoked after the provided fn exits (even if the
// process is interrupted by an OS termination signal). Notifications are only invoked once
// per Handler instance, so calling Run more than once will not behave as the user expects.
func (h *Handler) Run(fn func() error) error {
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, terminationSignals...)
	defer func() {
		signal.Stop(ch)
		close(ch)
	}()
	go func() {
		sig, ok := <-ch
		if !ok {
			return
		}
		h.Signal(sig)
	}()
	defer h.Close()
	return fn()
}


Validate

// Validate checks if a given ipset is valid or not.
func (set *IPSet) Validate() error {
	// Check if protocol is valid for `HashIPPort`, `HashIPPortIP` and `HashIPPortNet` type set.
	if set.SetType == HashIPPort || set.SetType == HashIPPortIP || set.SetType == HashIPPortNet {
		if err := validateHashFamily(set.HashFamily); err != nil {
			return err
		}
	}
	// check set type
	if err := validateIPSetType(set.SetType); err != nil {
		return err
	}
	// check port range for bitmap type set
	if set.SetType == BitmapPort {
		if err := validatePortRange(set.PortRange); err != nil {
			return err
		}
	}
	// check hash size value of ipset
	if set.HashSize <= 0 {
		return fmt.Errorf("invalid HashSize: %d", set.HashSize)
	}
	// check max elem value of ipset
	if set.MaxElem <= 0 {
		return fmt.Errorf("invalid MaxElem %d", set.MaxElem)
	}

	return nil
}


setIPSetDefaults

// setIPSetDefaults sets some IPSet fields if not present to their default values.
func (set *IPSet) setIPSetDefaults() {
	// Setting default values if not present
	if set.HashSize == 0 {
		set.HashSize = 1024
	}
	if set.MaxElem == 0 {
		set.MaxElem = 65536
	}
	// Default protocol is IPv4
	if set.HashFamily == "" {
		set.HashFamily = ProtocolFamilyIPV4
	}
	// Default ipset type is "hash:ip,port"
	if len(set.SetType) == 0 {
		set.SetType = HashIPPort
	}
	if len(set.PortRange) == 0 {
		set.PortRange = DefaultPortRange
	}
}


Validate

// Validate checks if a given ipset entry is valid or not.  The set parameter is the ipset that entry belongs to.
func (e *Entry) Validate(set *IPSet) bool {
	if e.Port < 0 {
		klog.Errorf("Entry %v port number %d should be >=0 for ipset %v", e, e.Port, set)
		return false
	}
	switch e.SetType {
	case HashIP:
		//check if IP of Entry is valid.
		if valid := e.checkIP(set); !valid {
			return false
		}
	case HashIPPort:
		//check if IP and Protocol of Entry is valid.
		if valid := e.checkIPandProtocol(set); !valid {
			return false
		}
	case HashIPPortIP:
		//check if IP and Protocol of Entry is valid.
		if valid := e.checkIPandProtocol(set); !valid {
			return false
		}

		// IP2 can not be empty for `hash:ip,port,ip` type ip set
		if netutils.ParseIPSloppy(e.IP2) == nil {
			klog.Errorf("Error parsing entry %v second ip address %v for ipset %v", e, e.IP2, set)
			return false
		}
	case HashIPPortNet:
		//check if IP and Protocol of Entry is valid.
		if valid := e.checkIPandProtocol(set); !valid {
			return false
		}

		// Net can not be empty for `hash:ip,port,net` type ip set
		if _, ipNet, err := netutils.ParseCIDRSloppy(e.Net); ipNet == nil {
			klog.Errorf("Error parsing entry %v ip net %v for ipset %v, error: %v", e, e.Net, set, err)
			return false
		}
	case BitmapPort:
		// check if port number satisfies its ipset's requirement of port range
		if set == nil {
			klog.Errorf("Unable to reference ip set where the entry %v exists", e)
			return false
		}
		begin, end, err := parsePortRange(set.PortRange)
		if err != nil {
			klog.Errorf("Failed to parse set %v port range %s for ipset %v, error: %v", set, set.PortRange, set, err)
			return false
		}
		if e.Port < begin || e.Port > end {
			klog.Errorf("Entry %v port number %d is not in the port range %s of its ipset %v", e, e.Port, set.PortRange, set)
			return false
		}
	}

	return true
}


checkIPandProtocol

// checkIPandProtocol checks if IP and Protocol of Entry is valid.
func (e *Entry) checkIPandProtocol(set *IPSet) bool {
	// set default protocol to tcp if empty
	if len(e.Protocol) == 0 {
		e.Protocol = ProtocolTCP
	} else if !validateProtocol(e.Protocol) {
		return false
	}
	return e.checkIP(set)
}


checkIP

// checkIP checks if IP of Entry is valid.
func (e *Entry) checkIP(set *IPSet) bool {
	if netutils.ParseIPSloppy(e.IP) == nil {
		klog.Errorf("Error parsing entry %v ip address %v for ipset %v", e, e.IP, set)
		return false
	}

	return true
}


New

// New returns a new Interface which will exec ipset.
func New(exec utilexec.Interface) Interface {
	return &runner{
		exec: exec,
	}
}


CreateSet

// CreateSet creates a new set, it will ignore error when the set already exists if ignoreExistErr=true.
func (runner *runner) CreateSet(set *IPSet, ignoreExistErr bool) error {
	// sets some IPSet fields if not present to their default values.
	set.setIPSetDefaults()

	// Validate ipset before creating
	if err := set.Validate(); err != nil {
		return err
	}
	return runner.createSet(set, ignoreExistErr)
}


createSet

// If ignoreExistErr is set to true, then the -exist option of ipset will be specified, ipset ignores the error
// otherwise raised when the same set (setname and create parameters are identical) already exists.
func (runner *runner) createSet(set *IPSet, ignoreExistErr bool) error {
	args := []string{"create", set.Name, string(set.SetType)}
	if set.SetType == HashIPPortIP || set.SetType == HashIPPort || set.SetType == HashIPPortNet || set.SetType == HashIP {
		args = append(args,
			"family", set.HashFamily,
			"hashsize", strconv.Itoa(set.HashSize),
			"maxelem", strconv.Itoa(set.MaxElem),
		)
	}
	if set.SetType == BitmapPort {
		args = append(args, "range", set.PortRange)
	}
	if ignoreExistErr {
		args = append(args, "-exist")
	}
	if _, err := runner.exec.Command(IPSetCmd, args...).CombinedOutput(); err != nil {
		return fmt.Errorf("error creating ipset %s, error: %v", set.Name, err)
	}
	return nil
}


AddEntry

// AddEntry adds a new entry to the named set.
// If the -exist option is specified, ipset ignores the error otherwise raised when
// the same set (setname and create parameters are identical) already exists.
func (runner *runner) AddEntry(entry string, set *IPSet, ignoreExistErr bool) error {
	args := []string{"add", set.Name, entry}
	if ignoreExistErr {
		args = append(args, "-exist")
	}
	if out, err := runner.exec.Command(IPSetCmd, args...).CombinedOutput(); err != nil {
		return fmt.Errorf("error adding entry %s, error: %v (%s)", entry, err, out)
	}
	return nil
}


DelEntry

// DelEntry is used to delete the specified entry from the set.
func (runner *runner) DelEntry(entry string, set string) error {
	if out, err := runner.exec.Command(IPSetCmd, "del", set, entry).CombinedOutput(); err != nil {
		return fmt.Errorf("error deleting entry %s: from set: %s, error: %v (%s)", entry, set, err, out)
	}
	return nil
}


TestEntry

// TestEntry is used to check whether the specified entry is in the set or not.
func (runner *runner) TestEntry(entry string, set string) (bool, error) {
	if out, err := runner.exec.Command(IPSetCmd, "test", set, entry).CombinedOutput(); err == nil {
		reg, e := regexp.Compile("is NOT in set " + set)
		if e == nil && reg.MatchString(string(out)) {
			return false, nil
		} else if e == nil {
			return true, nil
		} else {
			return false, fmt.Errorf("error testing entry: %s, error: %v", entry, e)
		}
	} else {
		return false, fmt.Errorf("error testing entry %s: %v (%s)", entry, err, out)
	}
}


FlushSet

// FlushSet deletes all entries from a named set.
func (runner *runner) FlushSet(set string) error {
	if _, err := runner.exec.Command(IPSetCmd, "flush", set).CombinedOutput(); err != nil {
		return fmt.Errorf("error flushing set: %s, error: %v", set, err)
	}
	return nil
}


DestroySet

// DestroySet is used to destroy a named set.
func (runner *runner) DestroySet(set string) error {
	if out, err := runner.exec.Command(IPSetCmd, "destroy", set).CombinedOutput(); err != nil {
		return fmt.Errorf("error destroying set %s, error: %v(%s)", set, err, out)
	}
	return nil
}


DestroyAllSets

// DestroyAllSets is used to destroy all sets.
func (runner *runner) DestroyAllSets() error {
	if _, err := runner.exec.Command(IPSetCmd, "destroy").CombinedOutput(); err != nil {
		return fmt.Errorf("error destroying all sets, error: %v", err)
	}
	return nil
}


ListSets

// ListSets list all set names from kernel
func (runner *runner) ListSets() ([]string, error) {
	out, err := runner.exec.Command(IPSetCmd, "list", "-n").CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("error listing all sets, error: %v", err)
	}
	return strings.Split(string(out), "\n"), nil
}


ListEntries

// ListEntries lists all the entries from a named set.
func (runner *runner) ListEntries(set string) ([]string, error) {
	if len(set) == 0 {
		return nil, fmt.Errorf("set name can't be nil")
	}
	out, err := runner.exec.Command(IPSetCmd, "list", set).CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("error listing set: %s, error: %v", set, err)
	}
	memberMatcher := regexp.MustCompile(EntryMemberPattern)
	list := memberMatcher.ReplaceAllString(string(out[:]), "")
	strs := strings.Split(list, "\n")
	results := make([]string, 0)
	for i := range strs {
		if len(strs[i]) > 0 {
			results = append(results, strs[i])
		}
	}
	return results, nil
}


GetVersion

// GetVersion returns the version string.
func (runner *runner) GetVersion() (string, error) {
	return getIPSetVersionString(runner.exec)
}


getIPSetVersionString

// getIPSetVersionString runs "ipset --version" to get the version string
// in the form of "X.Y", i.e "6.19"
func getIPSetVersionString(exec utilexec.Interface) (string, error) {
	cmd := exec.Command(IPSetCmd, "--version")
	cmd.SetStdin(bytes.NewReader([]byte{}))
	bytes, err := cmd.CombinedOutput()
	if err != nil {
		return "", err
	}
	versionMatcher := regexp.MustCompile(VersionPattern)
	match := versionMatcher.FindStringSubmatch(string(bytes))
	if match == nil {
		return "", fmt.Errorf("no ipset version found in string: %s", bytes)
	}
	return match[0], nil
}


validatePortRange

// checks if port range is valid. The begin port number is not necessarily less than
// end port number - ipset util can accept it.  It means both 1-100 and 100-1 are valid.
func validatePortRange(portRange string) error {
	strs := strings.Split(portRange, "-")
	if len(strs) != 2 {
		return fmt.Errorf("invalid PortRange: %q", portRange)
	}
	for i := range strs {
		num, err := strconv.Atoi(strs[i])
		if err != nil {
			return fmt.Errorf("invalid PortRange: %q", portRange)
		}
		if num < 0 {
			return fmt.Errorf("invalid PortRange: %q", portRange)
		}
	}
	return nil
}


validateIPSetType

// checks if the given ipset type is valid.
func validateIPSetType(set Type) error {
	for _, valid := range ValidIPSetTypes {
		if set == valid {
			return nil
		}
	}
	return fmt.Errorf("unsupported SetType: %q", set)
}


validateHashFamily

// checks if given hash family is supported in ipset
func validateHashFamily(family string) error {
	if family == ProtocolFamilyIPV4 || family == ProtocolFamilyIPV6 {
		return nil
	}
	return fmt.Errorf("unsupported HashFamily %q", family)
}


IsNotFoundError

// IsNotFoundError returns true if the error indicates "not found".  It parses
// the error string looking for known values, which is imperfect but works in
// practice.
func IsNotFoundError(err error) bool {
	es := err.Error()
	if strings.Contains(es, "does not exist") {
		// set with the same name already exists
		// xref: https://github.com/Olipro/ipset/blob/master/lib/errcode.c#L32-L33
		return true
	}
	if strings.Contains(es, "element is missing") {
		// entry is missing from the set
		// xref: https://github.com/Olipro/ipset/blob/master/lib/parse.c#L1904
		// https://github.com/Olipro/ipset/blob/master/lib/parse.c#L1925
		return true
	}
	return false
}


validateProtocol

// checks if given protocol is supported in entry
func validateProtocol(protocol string) bool {
	if protocol == ProtocolTCP || protocol == ProtocolUDP || protocol == ProtocolSCTP {
		return true
	}
	klog.Errorf("Invalid entry's protocol: %s, supported protocols are [%s, %s, %s]", protocol, ProtocolTCP, ProtocolUDP, ProtocolSCTP)
	return false
}