func (hv *WeightedHistogramVec) GetMetricWith(labels prometheus.Labels) (WeightedObserver, error) {
	metric, err := hv.MetricVec.GetMetricWith(labels)
	if metric != nil {
		return metric.(WeightedObserver), err
	}
	return nil, err
}


func (hv *WeightedHistogramVec) GetMetricWithLabelValues(lvs ...string) (WeightedObserver, error) {
	metric, err := hv.MetricVec.GetMetricWithLabelValues(lvs...)
	if metric != nil {
		return metric.(WeightedObserver), err
	}
	return nil, err
}


func (hv *WeightedHistogramVec) With(labels prometheus.Labels) WeightedObserver {
	h, err := hv.GetMetricWith(labels)
	if err != nil {
		panic(err)
	}
	return h
}


func (hv *WeightedHistogramVec) WithLabelValues(lvs ...string) WeightedObserver {
	h, err := hv.GetMetricWithLabelValues(lvs...)
	if err != nil {
		panic(err)
	}
	return h
}


func (hv *WeightedHistogramVec) CurryWith(labels prometheus.Labels) (WeightedObserverVec, error) {
	vec, err := hv.MetricVec.CurryWith(labels)
	if vec != nil {
		return &WeightedHistogramVec{MetricVec: vec}, err
	}
	return nil, err
}


func (hv *WeightedHistogramVec) MustCurryWith(labels prometheus.Labels) WeightedObserverVec {
	vec, err := hv.CurryWith(labels)
	if err != nil {
		panic(err)
	}
	return vec
}


// shouldHide be used to check if a specific metric with deprecated version should be hidden
// according to metrics deprecation lifecycle.
func shouldHide(currentVersion *semver.Version, deprecatedVersion *semver.Version) bool {
	guardVersion, err := semver.Make(fmt.Sprintf("%d.%d.0", currentVersion.Major, currentVersion.Minor))
	if err != nil {
		panic("failed to make version from current version")
	}

	if deprecatedVersion.LT(guardVersion) {
		return true
	}

	return false
}


// ValidateShowHiddenMetricsVersion checks invalid version for which show hidden metrics.
func ValidateShowHiddenMetricsVersion(v string) []error {
	err := validateShowHiddenMetricsVersion(parseVersion(version.Get()), v)
	if err != nil {
		return []error{err}
	}

	return nil
}


func SetDisabledMetric(name string)

func SetDisabledMetric(name string) {
	disabledMetricsLock.Lock()
	defer disabledMetricsLock.Unlock()
	disabledMetrics[name] = struct{}{}
	disabledMetricsTotal.Inc()
}


func SetShowHidden()

// SetShowHidden will enable showing hidden metrics. This will no-opt
// after the initial call
func SetShowHidden() {
	showHiddenOnce.Do(func() {
		showHidden.Store(true)

		// re-register collectors that has been hidden in phase of last registry.
		for _, r := range registries {
			r.enableHiddenCollectors()
			r.enableHiddenStableCollectors()
		}
	})
}


func ShouldShowHidden() bool

// ShouldShowHidden returns whether showing hidden deprecated metrics
// is enabled. While the primary usecase for this is internal (to determine
// registration behavior) this can also be used to introspect
func ShouldShowHidden() bool {
	return showHidden.Load()
}


// Register registers a new Collector to be included in metrics
// collection. It returns an error if the descriptors provided by the
// Collector are invalid or if they — in combination with descriptors of
// already registered Collectors — do not fulfill the consistency and
// uniqueness criteria described in the documentation of metric.Desc.
func (kr *kubeRegistry) Register(c Registerable) error {
	if c.Create(&kr.version) {
		defer kr.addResettable(c)
		return kr.PromRegistry.Register(c)
	}

	kr.trackHiddenCollector(c)
	return nil
}


// Registerer exposes the underlying prometheus.Registerer
func (kr *kubeRegistry) Registerer() prometheus.Registerer {
	return kr.PromRegistry
}


// Gatherer exposes the underlying prometheus.Gatherer
func (kr *kubeRegistry) Gatherer() prometheus.Gatherer {
	return kr.PromRegistry
}


// MustRegister works like Register but registers any number of
// Collectors and panics upon the first registration that causes an
// error.
func (kr *kubeRegistry) MustRegister(cs ...Registerable) {
	metrics := make([]prometheus.Collector, 0, len(cs))
	for _, c := range cs {
		if c.Create(&kr.version) {
			metrics = append(metrics, c)
			kr.addResettable(c)
		} else {
			kr.trackHiddenCollector(c)
		}
	}
	kr.PromRegistry.MustRegister(metrics...)
}


// CustomRegister registers a new custom collector.
func (kr *kubeRegistry) CustomRegister(c StableCollector) error {
	kr.trackStableCollectors(c)
	defer kr.addResettable(c)
	if c.Create(&kr.version, c) {
		return kr.PromRegistry.Register(c)
	}
	return nil
}


// CustomMustRegister works like CustomRegister but registers any number of
// StableCollectors and panics upon the first registration that causes an
// error.
func (kr *kubeRegistry) CustomMustRegister(cs ...StableCollector) {
	kr.trackStableCollectors(cs...)
	collectors := make([]prometheus.Collector, 0, len(cs))
	for _, c := range cs {
		if c.Create(&kr.version, c) {
			kr.addResettable(c)
			collectors = append(collectors, c)
		}
	}
	kr.PromRegistry.MustRegister(collectors...)
}


// RawMustRegister takes a native prometheus.Collector and registers the collector
// to the registry. This bypasses metrics safety checks, so should only be used
// to register custom prometheus collectors.
//
// Deprecated
func (kr *kubeRegistry) RawMustRegister(cs ...prometheus.Collector) {
	kr.PromRegistry.MustRegister(cs...)
	for _, c := range cs {
		kr.addResettable(c)
	}
}


// addResettable will automatically add our metric to our reset
// list if it satisfies the interface
func (kr *kubeRegistry) addResettable(i interface{}) {
	kr.resetLock.Lock()
	defer kr.resetLock.Unlock()
	if resettable, ok := i.(resettable); ok {
		kr.resettables = append(kr.resettables, resettable)
	}
}


// Unregister unregisters the Collector that equals the Collector passed
// in as an argument.  (Two Collectors are considered equal if their
// Describe method yields the same set of descriptors.) The function
// returns whether a Collector was unregistered. Note that an unchecked
// Collector cannot be unregistered (as its Describe method does not
// yield any descriptor).
func (kr *kubeRegistry) Unregister(collector Collector) bool {
	return kr.PromRegistry.Unregister(collector)
}


// Gather calls the Collect method of the registered Collectors and then
// gathers the collected metrics into a lexicographically sorted slice
// of uniquely named MetricFamily protobufs. Gather ensures that the
// returned slice is valid and self-consistent so that it can be used
// for valid exposition. As an exception to the strict consistency
// requirements described for metric.Desc, Gather will tolerate
// different sets of label names for metrics of the same metric family.
func (kr *kubeRegistry) Gather() ([]*dto.MetricFamily, error) {
	return kr.PromRegistry.Gather()
}


// trackHiddenCollector stores all hidden collectors.
func (kr *kubeRegistry) trackHiddenCollector(c Registerable) {
	kr.hiddenCollectorsLock.Lock()
	defer kr.hiddenCollectorsLock.Unlock()

	kr.hiddenCollectors[c.FQName()] = c
	hiddenMetricsTotal.Inc()
}


// trackStableCollectors stores all custom collectors.
func (kr *kubeRegistry) trackStableCollectors(cs ...StableCollector) {
	kr.stableCollectorsLock.Lock()
	defer kr.stableCollectorsLock.Unlock()

	kr.stableCollectors = append(kr.stableCollectors, cs...)
}


// enableHiddenCollectors will re-register all of the hidden collectors.
func (kr *kubeRegistry) enableHiddenCollectors() {
	if len(kr.hiddenCollectors) == 0 {
		return
	}

	kr.hiddenCollectorsLock.Lock()
	cs := make([]Registerable, 0, len(kr.hiddenCollectors))

	for _, c := range kr.hiddenCollectors {
		c.ClearState()
		cs = append(cs, c)
	}

	kr.hiddenCollectors = make(map[string]Registerable)
	kr.hiddenCollectorsLock.Unlock()
	kr.MustRegister(cs...)
}


// enableHiddenStableCollectors will re-register the stable collectors if there is one or more hidden metrics in it.
// Since we can not register a metrics twice, so we have to unregister first then register again.
func (kr *kubeRegistry) enableHiddenStableCollectors() {
	if len(kr.stableCollectors) == 0 {
		return
	}

	kr.stableCollectorsLock.Lock()

	cs := make([]StableCollector, 0, len(kr.stableCollectors))
	for _, c := range kr.stableCollectors {
		if len(c.HiddenMetrics()) > 0 {
			kr.Unregister(c) // unregister must happens before clear state, otherwise no metrics would be unregister
			c.ClearState()
			cs = append(cs, c)
		}
	}

	kr.stableCollectors = nil
	kr.stableCollectorsLock.Unlock()
	kr.CustomMustRegister(cs...)
}


func (kr *kubeRegistry) Reset()

// Reset invokes Reset on all metrics that are resettable.
func (kr *kubeRegistry) Reset() {
	kr.resetLock.RLock()
	defer kr.resetLock.RUnlock()
	for _, r := range kr.resettables {
		r.Reset()
	}
}


func newKubeRegistry(v apimachineryversion.Info) *kubeRegistry {
	r := &kubeRegistry{
		PromRegistry:     prometheus.NewRegistry(),
		version:          parseVersion(v),
		hiddenCollectors: make(map[string]Registerable),
		resettables:      make([]resettable, 0),
	}

	registriesLock.Lock()
	defer registriesLock.Unlock()
	registries = append(registries, r)

	return r
}


func NewKubeRegistry() KubeRegistry

// NewKubeRegistry creates a new vanilla Registry
func NewKubeRegistry() KubeRegistry {
	r := newKubeRegistry(BuildVersion())
	return r
}


func (r *kubeRegistry) RegisterMetaMetrics() {
	r.MustRegister(registeredMetrics)
	r.MustRegister(disabledMetricsTotal)
	r.MustRegister(hiddenMetricsTotal)
}


// NewSummary returns an object which is Summary-like. However, nothing
// will be measured until the summary is registered somewhere.
//
// DEPRECATED: as per the metrics overhaul KEP
func NewSummary(opts *SummaryOpts) *Summary {
	opts.StabilityLevel.setDefaults()

	s := &Summary{
		SummaryOpts: opts,
		lazyMetric:  lazyMetric{stabilityLevel: opts.StabilityLevel},
	}
	s.setPrometheusSummary(noopMetric{})
	s.lazyInit(s, BuildFQName(opts.Namespace, opts.Subsystem, opts.Name))
	return s
}


// setPrometheusSummary sets the underlying KubeGauge object, i.e. the thing that does the measurement.
func (s *Summary) setPrometheusSummary(summary prometheus.Summary) {
	s.ObserverMetric = summary
	s.initSelfCollection(summary)
}


// DeprecatedVersion returns a pointer to the Version or nil
func (s *Summary) DeprecatedVersion() *semver.Version {
	return parseSemver(s.SummaryOpts.DeprecatedVersion)
}


func (s *Summary) initializeMetric()

// initializeMetric invokes the actual prometheus.Summary object instantiation
// and stores a reference to it
func (s *Summary) initializeMetric() {
	s.SummaryOpts.annotateStabilityLevel()
	// this actually creates the underlying prometheus gauge.
	s.setPrometheusSummary(prometheus.NewSummary(s.SummaryOpts.toPromSummaryOpts()))
}


// initializeDeprecatedMetric invokes the actual prometheus.Summary object instantiation
// but modifies the Help description prior to object instantiation.
func (s *Summary) initializeDeprecatedMetric() {
	s.SummaryOpts.markDeprecated()
	s.initializeMetric()
}


// WithContext allows the normal Summary metric to pass in context. The context is no-op now.
func (s *Summary) WithContext(ctx context.Context) ObserverMetric {
	return s.ObserverMetric
}


// NewSummaryVec returns an object which satisfies kubeCollector and wraps the
// prometheus.SummaryVec object. However, the object returned will not measure
// anything unless the collector is first registered, since the metric is lazily instantiated,
// and only members extracted after
// registration will actually measure anything.
//
// DEPRECATED: as per the metrics overhaul KEP
func NewSummaryVec(opts *SummaryOpts, labels []string) *SummaryVec {
	opts.StabilityLevel.setDefaults()

	fqName := BuildFQName(opts.Namespace, opts.Subsystem, opts.Name)
	allowListLock.RLock()
	if allowList, ok := labelValueAllowLists[fqName]; ok {
		opts.LabelValueAllowLists = allowList
	}
	allowListLock.RUnlock()

	v := &SummaryVec{
		SummaryOpts:    opts,
		originalLabels: labels,
		lazyMetric:     lazyMetric{stabilityLevel: opts.StabilityLevel},
	}
	v.lazyInit(v, fqName)
	return v
}


// DeprecatedVersion returns a pointer to the Version or nil
func (v *SummaryVec) DeprecatedVersion() *semver.Version {
	return parseSemver(v.SummaryOpts.DeprecatedVersion)
}


func (v *SummaryVec) initializeMetric()

func (v *SummaryVec) initializeMetric() {
	v.SummaryOpts.annotateStabilityLevel()
	v.SummaryVec = prometheus.NewSummaryVec(v.SummaryOpts.toPromSummaryOpts(), v.originalLabels)
}


func (v *SummaryVec) initializeDeprecatedMetric() {
	v.SummaryOpts.markDeprecated()
	v.initializeMetric()
}


// WithLabelValues returns the ObserverMetric for the given slice of label
// values (same order as the VariableLabels in Desc). If that combination of
// label values is accessed for the first time, a new ObserverMetric is created IFF the summaryVec
// has been registered to a metrics registry.
func (v *SummaryVec) WithLabelValues(lvs ...string) ObserverMetric {
	if !v.IsCreated() {
		return noop
	}
	if v.LabelValueAllowLists != nil {
		v.LabelValueAllowLists.ConstrainToAllowedList(v.originalLabels, lvs)
	}
	return v.SummaryVec.WithLabelValues(lvs...)
}


// With returns the ObserverMetric for the given Labels map (the label names
// must match those of the VariableLabels in Desc). If that label map is
// accessed for the first time, a new ObserverMetric is created IFF the summaryVec has
// been registered to a metrics registry.
func (v *SummaryVec) With(labels map[string]string) ObserverMetric {
	if !v.IsCreated() {
		return noop
	}
	if v.LabelValueAllowLists != nil {
		v.LabelValueAllowLists.ConstrainLabelMap(labels)
	}
	return v.SummaryVec.With(labels)
}


// Delete deletes the metric where the variable labels are the same as those
// passed in as labels. It returns true if a metric was deleted.
//
// It is not an error if the number and names of the Labels are inconsistent
// with those of the VariableLabels in Desc. However, such inconsistent Labels
// can never match an actual metric, so the method will always return false in
// that case.
func (v *SummaryVec) Delete(labels map[string]string) bool {
	if !v.IsCreated() {
		return false // since we haven't created the metric, we haven't deleted a metric with the passed in values
	}
	return v.SummaryVec.Delete(labels)
}


func (v *SummaryVec) Reset()

// Reset deletes all metrics in this vector.
func (v *SummaryVec) Reset() {
	if !v.IsCreated() {
		return
	}

	v.SummaryVec.Reset()
}


// WithContext returns wrapped SummaryVec with context
func (v *SummaryVec) WithContext(ctx context.Context) *SummaryVecWithContext {
	return &SummaryVecWithContext{
		ctx:        ctx,
		SummaryVec: v,
	}
}


// WithLabelValues is the wrapper of SummaryVec.WithLabelValues.
func (vc *SummaryVecWithContext) WithLabelValues(lvs ...string) ObserverMetric {
	return vc.SummaryVec.WithLabelValues(lvs...)
}


// With is the wrapper of SummaryVec.With.
func (vc *SummaryVecWithContext) With(labels map[string]string) ObserverMetric {
	return vc.SummaryVec.With(labels)
}


// NewTimingHistogram returns an object which is TimingHistogram-like. However, nothing
// will be measured until the histogram is registered somewhere.
func NewTimingHistogram(opts *TimingHistogramOpts) *TimingHistogram {
	return NewTestableTimingHistogram(time.Now, opts)
}


// NewTestableTimingHistogram adds injection of the clock
func NewTestableTimingHistogram(nowFunc func() time.Time, opts *TimingHistogramOpts) *TimingHistogram {
	opts.StabilityLevel.setDefaults()

	h := &TimingHistogram{
		TimingHistogramOpts: opts,
		nowFunc:             nowFunc,
		lazyMetric:          lazyMetric{stabilityLevel: opts.StabilityLevel},
	}
	h.setPrometheusHistogram(noopMetric{})
	h.lazyInit(h, BuildFQName(opts.Namespace, opts.Subsystem, opts.Name))
	return h
}


// setPrometheusHistogram sets the underlying KubeGauge object, i.e. the thing that does the measurement.
func (h *TimingHistogram) setPrometheusHistogram(histogram promext.TimingHistogram) {
	h.PrometheusTimingHistogram = histogram
	h.initSelfCollection(histogram)
}


// DeprecatedVersion returns a pointer to the Version or nil
func (h *TimingHistogram) DeprecatedVersion() *semver.Version {
	return parseSemver(h.TimingHistogramOpts.DeprecatedVersion)
}


// initializeMetric invokes the actual prometheus.Histogram object instantiation
// and stores a reference to it
func (h *TimingHistogram) initializeMetric() {
	h.TimingHistogramOpts.annotateStabilityLevel()
	// this actually creates the underlying prometheus gauge.
	histogram, err := promext.NewTestableTimingHistogram(h.nowFunc, h.TimingHistogramOpts.toPromHistogramOpts())
	if err != nil {
		panic(err) // handle as for regular histograms
	}
	h.setPrometheusHistogram(histogram)
}


// initializeDeprecatedMetric invokes the actual prometheus.Histogram object instantiation
// but modifies the Help description prior to object instantiation.
func (h *TimingHistogram) initializeDeprecatedMetric() {
	h.TimingHistogramOpts.markDeprecated()
	h.initializeMetric()
}


// WithContext allows the normal TimingHistogram metric to pass in context. The context is no-op now.
func (h *TimingHistogram) WithContext(ctx context.Context) GaugeMetric {
	return h.PrometheusTimingHistogram
}


// NewTimingHistogramVec returns an object which satisfies the kubeCollector, Registerable, and GaugeVecMetric interfaces
// and wraps an underlying promext.TimingHistogramVec object.  Note well the way that
// behavior depends on registration and whether this is hidden.
func NewTimingHistogramVec(opts *TimingHistogramOpts, labels []string) *TimingHistogramVec {
	return NewTestableTimingHistogramVec(time.Now, opts, labels)
}


// NewTestableTimingHistogramVec adds injection of the clock.
func NewTestableTimingHistogramVec(nowFunc func() time.Time, opts *TimingHistogramOpts, labels []string) *TimingHistogramVec {
	opts.StabilityLevel.setDefaults()

	fqName := BuildFQName(opts.Namespace, opts.Subsystem, opts.Name)
	allowListLock.RLock()
	if allowList, ok := labelValueAllowLists[fqName]; ok {
		opts.LabelValueAllowLists = allowList
	}
	allowListLock.RUnlock()

	v := &TimingHistogramVec{
		TimingHistogramVec:  noopTimingHistogramVec,
		TimingHistogramOpts: opts,
		nowFunc:             nowFunc,
		originalLabels:      labels,
		lazyMetric:          lazyMetric{stabilityLevel: opts.StabilityLevel},
	}
	v.lazyInit(v, fqName)
	return v
}


// DeprecatedVersion returns a pointer to the Version or nil
func (v *TimingHistogramVec) DeprecatedVersion() *semver.Version {
	return parseSemver(v.TimingHistogramOpts.DeprecatedVersion)
}


func (v *TimingHistogramVec) initializeMetric() {
	v.TimingHistogramOpts.annotateStabilityLevel()
	v.TimingHistogramVec = promext.NewTestableTimingHistogramVec(v.nowFunc, v.TimingHistogramOpts.toPromHistogramOpts(), v.originalLabels...)
}


func (v *TimingHistogramVec) initializeDeprecatedMetric() {
	v.TimingHistogramOpts.markDeprecated()
	v.initializeMetric()
}


// WithLabelValuesChecked, if called before this vector has been registered in
// at least one registry, will return a noop gauge and
// an error that passes ErrIsNotRegistered.
// If called on a hidden vector,
// will return a noop gauge and a nil error.
// If called with a syntactic problem in the labels, will
// return a noop gauge and an error about the labels.
// If none of the above apply, this method will return
// the appropriate vector member and a nil error.
func (v *TimingHistogramVec) WithLabelValuesChecked(lvs ...string) (GaugeMetric, error) {
	if !v.IsCreated() {
		if v.IsHidden() {
			return noop, nil
		}
		return noop, errNotRegistered
	}
	if v.LabelValueAllowLists != nil {
		v.LabelValueAllowLists.ConstrainToAllowedList(v.originalLabels, lvs)
	}
	ops, err := v.TimingHistogramVec.GetMetricWithLabelValues(lvs...)
	if err != nil {
		return noop, err
	}
	return ops.(GaugeMetric), err
}


// WithLabelValues calls WithLabelValuesChecked
// and handles errors as follows.
// An error that passes ErrIsNotRegistered is ignored
// and the noop gauge is returned;
// all other errors cause a panic.
func (v *TimingHistogramVec) WithLabelValues(lvs ...string) GaugeMetric {
	ans, err := v.WithLabelValuesChecked(lvs...)
	if err == nil || ErrIsNotRegistered(err) {
		return ans
	}
	panic(err)
}


// WithChecked, if called before this vector has been registered in
// at least one registry, will return a noop gauge and
// an error that passes ErrIsNotRegistered.
// If called on a hidden vector,
// will return a noop gauge and a nil error.
// If called with a syntactic problem in the labels, will
// return a noop gauge and an error about the labels.
// If none of the above apply, this method will return
// the appropriate vector member and a nil error.
func (v *TimingHistogramVec) WithChecked(labels map[string]string) (GaugeMetric, error) {
	if !v.IsCreated() {
		if v.IsHidden() {
			return noop, nil
		}
		return noop, errNotRegistered
	}
	if v.LabelValueAllowLists != nil {
		v.LabelValueAllowLists.ConstrainLabelMap(labels)
	}
	ops, err := v.TimingHistogramVec.GetMetricWith(labels)
	return ops.(GaugeMetric), err
}


// With calls WithChecked and handles errors as follows.
// An error that passes ErrIsNotRegistered is ignored
// and the noop gauge is returned;
// all other errors cause a panic.
func (v *TimingHistogramVec) With(labels map[string]string) GaugeMetric {
	ans, err := v.WithChecked(labels)
	if err == nil || ErrIsNotRegistered(err) {
		return ans
	}
	panic(err)
}


// Delete deletes the metric where the variable labels are the same as those
// passed in as labels. It returns true if a metric was deleted.
//
// It is not an error if the number and names of the Labels are inconsistent
// with those of the VariableLabels in Desc. However, such inconsistent Labels
// can never match an actual metric, so the method will always return false in
// that case.
func (v *TimingHistogramVec) Delete(labels map[string]string) bool {
	if !v.IsCreated() {
		return false // since we haven't created the metric, we haven't deleted a metric with the passed in values
	}
	return v.TimingHistogramVec.Delete(labels)
}


func (v *TimingHistogramVec) Reset()

// Reset deletes all metrics in this vector.
func (v *TimingHistogramVec) Reset() {
	if !v.IsCreated() {
		return
	}

	v.TimingHistogramVec.Reset()
}


// WithContext returns wrapped TimingHistogramVec with context
func (v *TimingHistogramVec) InterfaceWithContext(ctx context.Context) GaugeVecMetric {
	return &TimingHistogramVecWithContext{
		ctx:                ctx,
		TimingHistogramVec: v,
	}
}


func (vt *ValueType) toPromValueType() prometheus.ValueType {
	return prometheus.ValueType(*vt)
}


// NewLazyConstMetric is a helper of MustNewConstMetric.
//
// Note: If the metrics described by the desc is hidden, the metrics will not be created.
func NewLazyConstMetric(desc *Desc, valueType ValueType, value float64, labelValues ...string) Metric {
	if desc.IsHidden() {
		return nil
	}
	return prometheus.MustNewConstMetric(desc.toPrometheusDesc(), valueType.toPromValueType(), value, labelValues...)
}


// NewConstMetric is a helper of NewConstMetric.
//
// Note: If the metrics described by the desc is hidden, the metrics will not be created.
func NewConstMetric(desc *Desc, valueType ValueType, value float64, labelValues ...string) (Metric, error) {
	if desc.IsHidden() {
		return nil, nil
	}
	return prometheus.NewConstMetric(desc.toPrometheusDesc(), valueType.toPromValueType(), value, labelValues...)
}


// NewLazyMetricWithTimestamp is a helper of NewMetricWithTimestamp.
//
// Warning: the Metric 'm' must be the one created by NewLazyConstMetric(),
// otherwise, no stability guarantees would be offered.
func NewLazyMetricWithTimestamp(t time.Time, m Metric) Metric {
	if m == nil {
		return nil
	}

	return prometheus.NewMetricWithTimestamp(t, m)
}


func RegisterBuildInfo(r KubeRegistry)

// RegisterBuildInfo registers the build and version info in a metadata metric in prometheus
func RegisterBuildInfo(r KubeRegistry) {
	info := version.Get()
	r.MustRegister(buildInfo)
	buildInfo.WithLabelValues(info.Major, info.Minor, info.GitVersion, info.GitCommit, info.GitTreeState, info.BuildDate, info.GoVersion, info.Compiler, info.Platform).Set(1)
}


func parseSemver(s string) *semver.Version {
	if s != "" {
		sv := semver.MustParse(s)
		return &sv
	}
	return nil
}


func parseVersion(ver apimachineryversion.Info) semver.Version {
	matches := versionRe.FindAllStringSubmatch(ver.String(), -1)

	if len(matches) != 1 {
		panic(fmt.Sprintf("version string \"%v\" doesn't match expected regular expression: \"%v\"", ver.String(), versionRe.String()))
	}
	return semver.MustParse(matches[0][1])
}


func ErrIsNotRegistered(err error) bool

func ErrIsNotRegistered(err error) bool {
	return err == errNotRegistered
}


// TerminalSize returns the current width and height of the user's terminal. If it isn't a terminal,
// nil is returned. On error, zero values are returned for width and height.
// Usually w must be the stdout of the process. Stderr won't work.
func TerminalSize(w io.Writer) (int, int, error) {
	outFd, isTerminal := term.GetFdInfo(w)
	if !isTerminal {
		return 0, 0, fmt.Errorf("given writer is no terminal")
	}
	winsize, err := term.GetWinsize(outFd)
	if err != nil {
		return 0, 0, err
	}
	return int(winsize.Width), int(winsize.Height), nil
}


// ValidateTracingConfiguration validates the tracing configuration
func ValidateTracingConfiguration(traceConfig *TracingConfiguration, featureGate featuregate.FeatureGate, fldPath *field.Path) field.ErrorList {
	allErrs := field.ErrorList{}
	if traceConfig == nil {
		return allErrs
	}
	if traceConfig.SamplingRatePerMillion != nil {
		allErrs = append(allErrs, validateSamplingRate(*traceConfig.SamplingRatePerMillion, fldPath.Child("samplingRatePerMillion"))...)
	}
	if traceConfig.Endpoint != nil {
		allErrs = append(allErrs, validateEndpoint(*traceConfig.Endpoint, fldPath.Child("endpoint"))...)
	}
	return allErrs
}


func validateSamplingRate(rate int32, fldPath *field.Path) field.ErrorList {
	errs := field.ErrorList{}
	if rate < 0 {
		errs = append(errs, field.Invalid(
			fldPath, rate,
			"sampling rate must be positive",
		))
	}
	if rate > maxSamplingRatePerMillion {
		errs = append(errs, field.Invalid(
			fldPath, rate,
			"sampling rate per million must be less than or equal to one million",
		))
	}
	return errs
}


func validateEndpoint(endpoint string, fldPath *field.Path) field.ErrorList {
	errs := field.ErrorList{}
	if !strings.Contains(endpoint, "//") {
		endpoint = "dns://" + endpoint
	}
	url, err := url.Parse(endpoint)
	if err != nil {
		errs = append(errs, field.Invalid(
			fldPath, endpoint,
			err.Error(),
		))
		return errs
	}
	switch url.Scheme {
	case "dns":
	case "unix":
	case "unix-abstract":
	default:
		errs = append(errs, field.Invalid(
			fldPath, endpoint,
			fmt.Sprintf("unsupported scheme: %v.  Options are none, dns, unix, or unix-abstract.  See https://github.com/grpc/grpc/blob/master/doc/naming.md", url.Scheme),
		))
	}
	return errs
}


// Start creates spans using both OpenTelemetry, and the k8s.io/utils/trace package.
// It only creates an OpenTelemetry span if the incoming context already includes a span.
func Start(ctx context.Context, name string, attributes ...attribute.KeyValue) (context.Context, *Span) {
	// If the incoming context already includes an OpenTelemetry span, create a child span with the provided name and attributes.
	// If the caller is not using OpenTelemetry, or has tracing disabled (e.g. with a component-specific feature flag), this is a noop.
	ctx, otelSpan := trace.SpanFromContext(ctx).TracerProvider().Tracer(instrumentationScope).Start(ctx, name, trace.WithAttributes(attributes...))
	// If there is already a utiltrace span in the context, use that as our parent span.
	utilSpan := utiltrace.FromContext(ctx).Nest(name, attributesToFields(attributes)...)
	// Set the trace as active in the context so that subsequent Start calls create nested spans.
	return utiltrace.ContextWithTrace(ctx, utilSpan), &Span{
		otelSpan: otelSpan,
		utilSpan: utilSpan,
	}
}


// AddEvent adds a point-in-time event with a name and attributes.
func (s *Span) AddEvent(name string, attributes ...attribute.KeyValue) {
	s.otelSpan.AddEvent(name, trace.WithAttributes(attributes...))
	if s.utilSpan != nil {
		s.utilSpan.Step(name, attributesToFields(attributes)...)
	}
}


// End ends the span, and logs if the span duration is greater than the logThreshold.
func (s *Span) End(logThreshold time.Duration) {
	s.otelSpan.End()
	if s.utilSpan != nil {
		s.utilSpan.LogIfLong(logThreshold)
	}
}


func attributesToFields(attributes []attribute.KeyValue) []utiltrace.Field {
	fields := make([]utiltrace.Field, len(attributes))
	for i := range attributes {
		attr := attributes[i]
		fields[i] = utiltrace.Field{Key: string(attr.Key), Value: attr.Value.AsInterface()}
	}
	return fields
}


// SpanFromContext returns the *Span from the current context. It is composed of the active
// OpenTelemetry and k8s.io/utils/trace spans.
func SpanFromContext(ctx context.Context) *Span {
	return &Span{
		otelSpan: trace.SpanFromContext(ctx),
		utilSpan: utiltrace.FromContext(ctx),
	}
}


// ContextWithSpan returns a context with the Span included in the context.
func ContextWithSpan(ctx context.Context, s *Span) context.Context {
	return trace.ContextWithSpan(utiltrace.ContextWithTrace(ctx, s.utilSpan), s.otelSpan)
}


func (n *noopTracerProvider) Shutdown(context.Context) error {
	return nil
}


func NewNoopTracerProvider() TracerProvider {
	return &noopTracerProvider{TracerProvider: oteltrace.NewNoopTracerProvider()}
}


func NewProvider(ctx context.Context,

// NewProvider creates a TracerProvider in a component, and enforces recommended tracing behavior
func NewProvider(ctx context.Context,
	tracingConfig *v1.TracingConfiguration,
	addedOpts []otlptracegrpc.Option,
	resourceOpts []resource.Option,
) (TracerProvider, error) {
	if tracingConfig == nil {
		return NewNoopTracerProvider(), nil
	}
	opts := append([]otlptracegrpc.Option{}, addedOpts...)
	if tracingConfig.Endpoint != nil {
		opts = append(opts, otlptracegrpc.WithEndpoint(*tracingConfig.Endpoint))
	}
	opts = append(opts, otlptracegrpc.WithInsecure())
	exporter, err := otlptracegrpc.New(ctx, opts...)
	if err != nil {
		return nil, err
	}
	res, err := resource.New(ctx, resourceOpts...)
	if err != nil {
		return nil, err
	}

	// sampler respects parent span's sampling rate or
	// otherwise never samples.
	sampler := sdktrace.NeverSample()
	// Or, emit spans for a fraction of transactions
	if tracingConfig.SamplingRatePerMillion != nil && *tracingConfig.SamplingRatePerMillion > 0 {
		sampler = sdktrace.TraceIDRatioBased(float64(*tracingConfig.SamplingRatePerMillion) / float64(1000000))
	}
	// batch span processor to aggregate spans before export.
	bsp := sdktrace.NewBatchSpanProcessor(exporter)
	tp := sdktrace.NewTracerProvider(
		sdktrace.WithSampler(sdktrace.ParentBased(sampler)),
		sdktrace.WithSpanProcessor(bsp),
		sdktrace.WithResource(res),
	)
	return tp, nil
}


// WithTracing adds tracing to requests if the incoming request is sampled
func WithTracing(handler http.Handler, tp oteltrace.TracerProvider, serviceName string) http.Handler {
	opts := []otelhttp.Option{
		otelhttp.WithPropagators(Propagators()),
		otelhttp.WithTracerProvider(tp),
	}
	// With Noop TracerProvider, the otelhttp still handles context propagation.
	// See https://github.com/open-telemetry/opentelemetry-go/tree/main/example/passthrough
	return otelhttp.NewHandler(handler, serviceName, opts...)
}


// WrapperFor can be used to add tracing to a *rest.Config.
// Example usage:
// tp := NewProvider(...)
// config, _ := rest.InClusterConfig()
// config.Wrap(WrapperFor(tp))
// kubeclient, _ := clientset.NewForConfig(config)
func WrapperFor(tp oteltrace.TracerProvider) transport.WrapperFunc {
	return func(rt http.RoundTripper) http.RoundTripper {
		opts := []otelhttp.Option{
			otelhttp.WithPropagators(Propagators()),
			otelhttp.WithTracerProvider(tp),
		}
		// With Noop TracerProvider, the otelhttp still handles context propagation.
		// See https://github.com/open-telemetry/opentelemetry-go/tree/main/example/passthrough
		return otelhttp.NewTransport(rt, opts...)
	}
}


// Propagators returns the recommended set of propagators.
func Propagators() propagation.TextMapPropagator {
	return propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})
}


func (v *versionValue) IsBoolFlag() bool {
	return true
}


func (v *versionValue) Get() interface{} {
	return versionValue(*v)
}


func (v *versionValue) Set(s string) error {
	if s == strRawVersion {
		*v = VersionRaw
		return nil
	}
	boolVal, err := strconv.ParseBool(s)
	if boolVal {
		*v = VersionTrue
	} else {
		*v = VersionFalse
	}
	return err
}


func (v *versionValue) String() string

func (v *versionValue) String() string {
	if *v == VersionRaw {
		return strRawVersion
	}
	return fmt.Sprintf("%v", bool(*v == VersionTrue))
}


func (v *versionValue) Type() string

// The type of the flag as required by the pflag.Value interface
func (v *versionValue) Type() string {
	return "version"
}


func VersionVar(p *versionValue, name string, value versionValue, usage string) {
	*p = value
	flag.Var(p, name, usage)
	// "--version" will be treated as "--version=true"
	flag.Lookup(name).NoOptDefVal = "true"
}


func Version(name string, value versionValue, usage string) *versionValue {
	p := new(versionValue)
	VersionVar(p, name, value, usage)
	return p
}


func AddFlags(fs *flag.FlagSet)

// AddFlags registers this package's flags on arbitrary FlagSets, such that they point to the
// same value as the global flags.
func AddFlags(fs *flag.FlagSet) {
	fs.AddFlag(flag.Lookup(versionFlagName))
}


func PrintAndExitIfRequested()

// PrintAndExitIfRequested will check if the -version flag was passed
// and, if so, print the version and exit.
func PrintAndExitIfRequested() {
	if *versionFlag == VersionRaw {
		fmt.Printf("%#v\n", version.Get())
		os.Exit(0)
	} else if *versionFlag == VersionTrue {
		fmt.Printf("%s %s\n", programName, version.Get())
		os.Exit(0)
	}
}


func Get() apimachineryversion.Info

// Get returns the overall codebase version. It's for detecting
// what code a binary was built from.
func Get() apimachineryversion.Info {
	// These variables typically come from -ldflags settings and in
	// their absence fallback to the settings in ./base.go
	return apimachineryversion.Info{
		Major:        gitMajor,
		Minor:        gitMinor,
		GitVersion:   gitVersion,
		GitCommit:    gitCommit,
		GitTreeState: gitTreeState,
		BuildDate:    buildDate,
		GoVersion:    runtime.Version(),
		Compiler:     runtime.Compiler,
		Platform:     fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH),
	}
}


// NewController constructs and returns a controller
func NewController(clock clock.Clock, client clientset.Interface, holderIdentity string, leaseDurationSeconds int32, onRepeatedHeartbeatFailure func(), renewInterval time.Duration, leaseName, leaseNamespace string, newLeasePostProcessFunc ProcessLeaseFunc) Controller {
	var leaseClient coordclientset.LeaseInterface
	if client != nil {
		leaseClient = client.CoordinationV1().Leases(leaseNamespace)
	}
	return &controller{
		client:                     client,
		leaseClient:                leaseClient,
		holderIdentity:             holderIdentity,
		leaseName:                  leaseName,
		leaseNamespace:             leaseNamespace,
		leaseDurationSeconds:       leaseDurationSeconds,
		renewInterval:              renewInterval,
		clock:                      clock,
		onRepeatedHeartbeatFailure: onRepeatedHeartbeatFailure,
		newLeasePostProcessFunc:    newLeasePostProcessFunc,
	}
}


// Run runs the controller
func (c *controller) Run(ctx context.Context) {
	if c.leaseClient == nil {
		klog.FromContext(ctx).Info("lease controller has nil lease client, will not claim or renew leases")
		return
	}
	wait.JitterUntilWithContext(ctx, c.sync, c.renewInterval, 0.04, true)
}


func (c *controller) sync(ctx context.Context) {
	if c.latestLease != nil {
		// As long as the lease is not (or very rarely) updated by any other agent than the component itself,
		// we can optimistically assume it didn't change since our last update and try updating
		// based on the version from that time. Thanks to it we avoid GET call and reduce load
		// on etcd and kube-apiserver.
		// If at some point other agents will also be frequently updating the Lease object, this
		// can result in performance degradation, because we will end up with calling additional
		// GET/PUT - at this point this whole "if" should be removed.
		err := c.retryUpdateLease(ctx, c.latestLease)
		if err == nil {
			return
		}
		klog.FromContext(ctx).Info("failed to update lease using latest lease, fallback to ensure lease", "err", err)
	}

	lease, created := c.backoffEnsureLease(ctx)
	c.latestLease = lease
	// we don't need to update the lease if we just created it
	if !created && lease != nil {
		if err := c.retryUpdateLease(ctx, lease); err != nil {
			klog.FromContext(ctx).Error(err, "Will retry updating lease", "interval", c.renewInterval)
		}
	}
}


// backoffEnsureLease attempts to create the lease if it does not exist,
// and uses exponentially increasing waits to prevent overloading the API server
// with retries. Returns the lease, and true if this call created the lease,
// false otherwise.
func (c *controller) backoffEnsureLease(ctx context.Context) (*coordinationv1.Lease, bool) {
	var (
		lease   *coordinationv1.Lease
		created bool
		err     error
	)
	sleep := 100 * time.Millisecond
	for {
		lease, created, err = c.ensureLease(ctx)
		if err == nil {
			break
		}
		sleep = minDuration(2*sleep, maxBackoff)
		klog.FromContext(ctx).Error(err, "Failed to ensure lease exists, will retry", "interval", sleep)
		// backoff wait with early return if the context gets canceled
		select {
		case <-ctx.Done():
			return nil, false
		case <-time.After(sleep):
		}
	}
	return lease, created
}


// ensureLease creates the lease if it does not exist. Returns the lease and
// a bool (true if this call created the lease), or any error that occurs.
func (c *controller) ensureLease(ctx context.Context) (*coordinationv1.Lease, bool, error) {
	lease, err := c.leaseClient.Get(ctx, c.leaseName, metav1.GetOptions{})
	if apierrors.IsNotFound(err) {
		// lease does not exist, create it.
		leaseToCreate, err := c.newLease(nil)
		// An error occurred during allocating the new lease (likely from newLeasePostProcessFunc).
		// Given that we weren't able to set the lease correctly, we simply
		// not create it this time - we will retry in the next iteration.
		if err != nil {
			return nil, false, nil
		}
		lease, err := c.leaseClient.Create(ctx, leaseToCreate, metav1.CreateOptions{})
		if err != nil {
			return nil, false, err
		}
		return lease, true, nil
	} else if err != nil {
		// unexpected error getting lease
		return nil, false, err
	}
	// lease already existed
	return lease, false, nil
}


// retryUpdateLease attempts to update the lease for maxUpdateRetries,
// call this once you're sure the lease has been created
func (c *controller) retryUpdateLease(ctx context.Context, base *coordinationv1.Lease) error {
	for i := 0; i < maxUpdateRetries; i++ {
		leaseToUpdate, _ := c.newLease(base)
		lease, err := c.leaseClient.Update(ctx, leaseToUpdate, metav1.UpdateOptions{})
		if err == nil {
			c.latestLease = lease
			return nil
		}
		klog.FromContext(ctx).Error(err, "Failed to update lease")
		// OptimisticLockError requires getting the newer version of lease to proceed.
		if apierrors.IsConflict(err) {
			base, _ = c.backoffEnsureLease(ctx)
			continue
		}
		if i > 0 && c.onRepeatedHeartbeatFailure != nil {
			c.onRepeatedHeartbeatFailure()
		}
	}
	return fmt.Errorf("failed %d attempts to update lease", maxUpdateRetries)
}


// newLease constructs a new lease if base is nil, or returns a copy of base
// with desired state asserted on the copy.
// Note that an error will block lease CREATE, causing the CREATE to be retried in
// the next iteration; but the error won't block lease refresh (UPDATE).
func (c *controller) newLease(base *coordinationv1.Lease) (*coordinationv1.Lease, error) {
	// Use the bare minimum set of fields; other fields exist for debugging/legacy,
	// but we don't need to make component heartbeats more complicated by using them.
	var lease *coordinationv1.Lease
	if base == nil {
		lease = &coordinationv1.Lease{
			ObjectMeta: metav1.ObjectMeta{
				Name:      c.leaseName,
				Namespace: c.leaseNamespace,
			},
			Spec: coordinationv1.LeaseSpec{
				HolderIdentity:       pointer.StringPtr(c.holderIdentity),
				LeaseDurationSeconds: pointer.Int32Ptr(c.leaseDurationSeconds),
			},
		}
	} else {
		lease = base.DeepCopy()
	}
	lease.Spec.RenewTime = &metav1.MicroTime{Time: c.clock.Now()}

	if c.newLeasePostProcessFunc != nil {
		err := c.newLeasePostProcessFunc(lease)
		return lease, err
	}

	return lease, nil
}


func minDuration(a, b time.Duration) time.Duration {
	if a < b {
		return a
	}
	return b
}


// UpdateDisruptionAllowedCondition updates the DisruptionAllowed condition
// on a PodDisruptionBudget based on the value of the DisruptionsAllowed field.
func UpdateDisruptionAllowedCondition(pdb *policy.PodDisruptionBudget) {
	if pdb.Status.Conditions == nil {
		pdb.Status.Conditions = make([]metav1.Condition, 0)
	}
	if pdb.Status.DisruptionsAllowed > 0 {
		apimeta.SetStatusCondition(&pdb.Status.Conditions, metav1.Condition{
			Type:               policy.DisruptionAllowedCondition,
			Reason:             policy.SufficientPodsReason,
			Status:             metav1.ConditionTrue,
			ObservedGeneration: pdb.Status.ObservedGeneration,
		})
	} else {
		apimeta.SetStatusCondition(&pdb.Status.Conditions, metav1.Condition{
			Type:               policy.DisruptionAllowedCondition,
			Reason:             policy.InsufficientPodsReason,
			Status:             metav1.ConditionFalse,
			ObservedGeneration: pdb.Status.ObservedGeneration,
		})
	}
}


// ConditionsAreUpToDate checks whether the status and reason for the
// DisruptionAllowed condition are set to the correct values based on the
// DisruptionsAllowed field.
func ConditionsAreUpToDate(pdb *policy.PodDisruptionBudget) bool {
	cond := apimeta.FindStatusCondition(pdb.Status.Conditions, policy.DisruptionAllowedCondition)
	if cond == nil {
		return false
	}

	if pdb.Status.ObservedGeneration != pdb.Generation {
		return false
	}

	if pdb.Status.DisruptionsAllowed > 0 {
		return cond.Status == metav1.ConditionTrue && cond.Reason == policy.SufficientPodsReason
	}
	return cond.Status == metav1.ConditionFalse && cond.Reason == policy.InsufficientPodsReason
}


func (o ClusterRoleRuleOwner) GetObject() runtime.Object {
	return o.ClusterRole
}


func (o ClusterRoleRuleOwner) GetNamespace() string {
	return o.ClusterRole.Namespace
}


func (o ClusterRoleRuleOwner) GetName() string {
	return o.ClusterRole.Name
}


func (o ClusterRoleRuleOwner) GetLabels() map[string]string {
	return o.ClusterRole.Labels
}


func (o ClusterRoleRuleOwner) SetLabels(in map[string]string) {
	o.ClusterRole.Labels = in
}


func (o ClusterRoleRuleOwner) GetAnnotations() map[string]string {
	return o.ClusterRole.Annotations
}


func (o ClusterRoleRuleOwner) SetAnnotations(in map[string]string) {
	o.ClusterRole.Annotations = in
}


func (o ClusterRoleRuleOwner) GetRules() []rbacv1.PolicyRule {
	return o.ClusterRole.Rules
}


func (o ClusterRoleRuleOwner) SetRules(in []rbacv1.PolicyRule) {
	o.ClusterRole.Rules = in
}


func (o ClusterRoleRuleOwner) GetAggregationRule() *rbacv1.AggregationRule {
	return o.ClusterRole.AggregationRule
}


func (o ClusterRoleRuleOwner) SetAggregationRule(in *rbacv1.AggregationRule) {
	o.ClusterRole.AggregationRule = in
}


func (c ClusterRoleModifier) Get(namespace, name string) (RuleOwner, error) {
	ret, err := c.Client.Get(context.TODO(), name, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}
	return ClusterRoleRuleOwner{ClusterRole: ret}, err
}


func (c ClusterRoleModifier) Create(in RuleOwner) (RuleOwner, error) {
	ret, err := c.Client.Create(context.TODO(), in.(ClusterRoleRuleOwner).ClusterRole, metav1.CreateOptions{})
	if err != nil {
		return nil, err
	}
	return ClusterRoleRuleOwner{ClusterRole: ret}, err
}


func (c ClusterRoleModifier) Update(in RuleOwner) (RuleOwner, error) {
	ret, err := c.Client.Update(context.TODO(), in.(ClusterRoleRuleOwner).ClusterRole, metav1.UpdateOptions{})
	if err != nil {
		return nil, err
	}
	return ClusterRoleRuleOwner{ClusterRole: ret}, err

}


func (o ClusterRoleBindingAdapter) GetObject() runtime.Object {
	return o.ClusterRoleBinding
}


func (o ClusterRoleBindingAdapter) GetNamespace() string {
	return o.ClusterRoleBinding.Namespace
}


func (o ClusterRoleBindingAdapter) GetName() string {
	return o.ClusterRoleBinding.Name
}


func (o ClusterRoleBindingAdapter) GetUID() types.UID {
	return o.ClusterRoleBinding.UID
}


func (o ClusterRoleBindingAdapter) GetLabels() map[string]string {
	return o.ClusterRoleBinding.Labels
}


func (o ClusterRoleBindingAdapter) SetLabels(in map[string]string) {
	o.ClusterRoleBinding.Labels = in
}


func (o ClusterRoleBindingAdapter) GetAnnotations() map[string]string {
	return o.ClusterRoleBinding.Annotations
}


func (o ClusterRoleBindingAdapter) SetAnnotations(in map[string]string) {
	o.ClusterRoleBinding.Annotations = in
}


func (o ClusterRoleBindingAdapter) GetRoleRef() rbacv1.RoleRef {
	return o.ClusterRoleBinding.RoleRef
}


func (o ClusterRoleBindingAdapter) GetSubjects() []rbacv1.Subject {
	return o.ClusterRoleBinding.Subjects
}


func (o ClusterRoleBindingAdapter) SetSubjects(in []rbacv1.Subject) {
	o.ClusterRoleBinding.Subjects = in
}


func (c ClusterRoleBindingClientAdapter) Get(namespace, name string) (RoleBinding, error) {
	ret, err := c.Client.Get(context.TODO(), name, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}
	return ClusterRoleBindingAdapter{ClusterRoleBinding: ret}, err
}


func (c ClusterRoleBindingClientAdapter) Create(in RoleBinding) (RoleBinding, error) {
	ret, err := c.Client.Create(context.TODO(), in.(ClusterRoleBindingAdapter).ClusterRoleBinding, metav1.CreateOptions{})
	if err != nil {
		return nil, err
	}
	return ClusterRoleBindingAdapter{ClusterRoleBinding: ret}, err
}


func (c ClusterRoleBindingClientAdapter) Update(in RoleBinding) (RoleBinding, error) {
	ret, err := c.Client.Update(context.TODO(), in.(ClusterRoleBindingAdapter).ClusterRoleBinding, metav1.UpdateOptions{})
	if err != nil {
		return nil, err
	}
	return ClusterRoleBindingAdapter{ClusterRoleBinding: ret}, err

}


func (c ClusterRoleBindingClientAdapter) Delete(namespace, name string, uid types.UID) error {
	return c.Client.Delete(context.TODO(), name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}})
}


// tryEnsureNamespace gets or creates the given namespace while ignoring forbidden errors.
// It is a best effort attempt as the user may not be able to get or create namespaces.
// This allows us to handle flows where the user can only mutate roles and role bindings.
func tryEnsureNamespace(client corev1client.NamespaceInterface, namespace string) error {
	_, getErr := client.Get(context.TODO(), namespace, metav1.GetOptions{})
	if getErr == nil {
		return nil
	}

	if fatalGetErr := utilerrors.FilterOut(getErr, apierrors.IsNotFound, apierrors.IsForbidden); fatalGetErr != nil {
		return fatalGetErr
	}

	ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}
	_, createErr := client.Create(context.TODO(), ns, metav1.CreateOptions{})

	return utilerrors.FilterOut(createErr, apierrors.IsAlreadyExists, apierrors.IsForbidden)
}


func (o *ReconcileRoleOptions) Run() (*ReconcileClusterRoleResult, error) {
	return o.run(0)
}


func (o *ReconcileRoleOptions) run(attempts int) (*ReconcileClusterRoleResult, error) {
	// This keeps us from retrying forever if a role keeps appearing and disappearing as we reconcile.
	// Conflict errors on update are handled at a higher level.
	if attempts > 2 {
		return nil, fmt.Errorf("exceeded maximum attempts")
	}

	var result *ReconcileClusterRoleResult

	existing, err := o.Client.Get(o.Role.GetNamespace(), o.Role.GetName())
	switch {
	case errors.IsNotFound(err):
		aggregationRule := o.Role.GetAggregationRule()
		if aggregationRule == nil {
			aggregationRule = &rbacv1.AggregationRule{}
		}
		result = &ReconcileClusterRoleResult{
			Role:                            o.Role,
			MissingRules:                    o.Role.GetRules(),
			MissingAggregationRuleSelectors: aggregationRule.ClusterRoleSelectors,
			Operation:                       ReconcileCreate,
		}

	case err != nil:
		return nil, err

	default:
		result, err = computeReconciledRole(existing, o.Role, o.RemoveExtraPermissions)
		if err != nil {
			return nil, err
		}
	}

	// If reconcile-protected, short-circuit
	if result.Protected {
		return result, nil
	}
	// If we're in dry-run mode, short-circuit
	if !o.Confirm {
		return result, nil
	}

	switch result.Operation {
	case ReconcileCreate:
		created, err := o.Client.Create(result.Role)
		// If created since we started this reconcile, re-run
		if errors.IsAlreadyExists(err) {
			return o.run(attempts + 1)
		}
		if err != nil {
			return nil, err
		}
		result.Role = created

	case ReconcileUpdate:
		updated, err := o.Client.Update(result.Role)
		// If deleted since we started this reconcile, re-run
		if errors.IsNotFound(err) {
			return o.run(attempts + 1)
		}
		if err != nil {
			return nil, err
		}
		result.Role = updated

	case ReconcileNone:
		// no-op

	default:
		return nil, fmt.Errorf("invalid operation: %v", result.Operation)
	}

	return result, nil
}


// computeReconciledRole returns the role that must be created and/or updated to make the
// existing role's permissions match the expected role's permissions
func computeReconciledRole(existing, expected RuleOwner, removeExtraPermissions bool) (*ReconcileClusterRoleResult, error) {
	result := &ReconcileClusterRoleResult{Operation: ReconcileNone}

	result.Protected = (existing.GetAnnotations()[rbacv1.AutoUpdateAnnotationKey] == "false")

	// Start with a copy of the existing object
	result.Role = existing.DeepCopyRuleOwner()

	// Merge expected annotations and labels
	result.Role.SetAnnotations(merge(expected.GetAnnotations(), result.Role.GetAnnotations()))
	if !reflect.DeepEqual(result.Role.GetAnnotations(), existing.GetAnnotations()) {
		result.Operation = ReconcileUpdate
	}
	result.Role.SetLabels(merge(expected.GetLabels(), result.Role.GetLabels()))
	if !reflect.DeepEqual(result.Role.GetLabels(), existing.GetLabels()) {
		result.Operation = ReconcileUpdate
	}

	// Compute extra and missing rules
	// Don't compute extra permissions if expected and existing roles are both aggregated
	if expected.GetAggregationRule() == nil || existing.GetAggregationRule() == nil {
		_, result.ExtraRules = validation.Covers(expected.GetRules(), existing.GetRules())
	}
	_, result.MissingRules = validation.Covers(existing.GetRules(), expected.GetRules())

	switch {
	case !removeExtraPermissions && len(result.MissingRules) > 0:
		// add missing rules in the union case
		result.Role.SetRules(append(result.Role.GetRules(), result.MissingRules...))
		result.Operation = ReconcileUpdate

	case removeExtraPermissions && (len(result.MissingRules) > 0 || len(result.ExtraRules) > 0):
		// stomp to expected rules in the non-union case
		result.Role.SetRules(expected.GetRules())
		result.Operation = ReconcileUpdate
	}

	// Compute extra and missing rules
	_, result.ExtraAggregationRuleSelectors = aggregationRuleCovers(expected.GetAggregationRule(), existing.GetAggregationRule())
	_, result.MissingAggregationRuleSelectors = aggregationRuleCovers(existing.GetAggregationRule(), expected.GetAggregationRule())

	switch {
	case expected.GetAggregationRule() == nil && existing.GetAggregationRule() != nil:
		// we didn't expect this to be an aggregated role at all, remove the existing aggregation
		result.Role.SetAggregationRule(nil)
		result.Operation = ReconcileUpdate

	case !removeExtraPermissions && len(result.MissingAggregationRuleSelectors) > 0:
		// add missing rules in the union case
		aggregationRule := result.Role.GetAggregationRule()
		if aggregationRule == nil {
			aggregationRule = &rbacv1.AggregationRule{}
		}
		aggregationRule.ClusterRoleSelectors = append(aggregationRule.ClusterRoleSelectors, result.MissingAggregationRuleSelectors...)
		result.Role.SetAggregationRule(aggregationRule)
		result.Operation = ReconcileUpdate

	case removeExtraPermissions && (len(result.MissingAggregationRuleSelectors) > 0 || len(result.ExtraAggregationRuleSelectors) > 0):
		result.Role.SetAggregationRule(expected.GetAggregationRule())
		result.Operation = ReconcileUpdate
	}

	return result, nil
}


// merge combines the given maps with the later annotations having higher precedence
func merge(maps ...map[string]string) map[string]string {
	var output map[string]string = nil
	for _, m := range maps {
		if m != nil && output == nil {
			output = map[string]string{}
		}
		for k, v := range m {
			output[k] = v
		}
	}
	return output
}


// aggregationRuleCovers determines whether or not the ownerSelectors cover the servantSelectors in terms of semantically
// equal label selectors.
// It returns whether or not the ownerSelectors cover and a list of the rules that the ownerSelectors do not cover.
func aggregationRuleCovers(ownerRule, servantRule *rbacv1.AggregationRule) (bool, []metav1.LabelSelector) {
	switch {
	case ownerRule == nil && servantRule == nil:
		return true, []metav1.LabelSelector{}
	case ownerRule == nil && servantRule != nil:
		return false, servantRule.ClusterRoleSelectors
	case ownerRule != nil && servantRule == nil:
		return true, []metav1.LabelSelector{}

	}

	ownerSelectors := ownerRule.ClusterRoleSelectors
	servantSelectors := servantRule.ClusterRoleSelectors
	uncoveredSelectors := []metav1.LabelSelector{}

	for _, servantSelector := range servantSelectors {
		covered := false
		for _, ownerSelector := range ownerSelectors {
			if equality.Semantic.DeepEqual(ownerSelector, servantSelector) {
				covered = true
				break
			}
		}
		if !covered {
			uncoveredSelectors = append(uncoveredSelectors, servantSelector)
		}
	}

	return (len(uncoveredSelectors) == 0), uncoveredSelectors
}


func (o *ReconcileRoleBindingOptions) Run() (*ReconcileClusterRoleBindingResult, error) {
	return o.run(0)
}


func (o *ReconcileRoleBindingOptions) run(attempts int) (*ReconcileClusterRoleBindingResult, error) {
	// This keeps us from retrying forever if a rolebinding keeps appearing and disappearing as we reconcile.
	// Conflict errors on update are handled at a higher level.
	if attempts > 3 {
		return nil, fmt.Errorf("exceeded maximum attempts")
	}

	var result *ReconcileClusterRoleBindingResult

	existingBinding, err := o.Client.Get(o.RoleBinding.GetNamespace(), o.RoleBinding.GetName())
	switch {
	case errors.IsNotFound(err):
		result = &ReconcileClusterRoleBindingResult{
			RoleBinding:     o.RoleBinding,
			MissingSubjects: o.RoleBinding.GetSubjects(),
			Operation:       ReconcileCreate,
		}

	case err != nil:
		return nil, err

	default:
		result, err = computeReconciledRoleBinding(existingBinding, o.RoleBinding, o.RemoveExtraSubjects)
		if err != nil {
			return nil, err
		}
	}

	// If reconcile-protected, short-circuit
	if result.Protected {
		return result, nil
	}
	// If we're in dry-run mode, short-circuit
	if !o.Confirm {
		return result, nil
	}

	switch result.Operation {
	case ReconcileRecreate:
		// Try deleting
		err := o.Client.Delete(existingBinding.GetNamespace(), existingBinding.GetName(), existingBinding.GetUID())
		switch {
		case err == nil, errors.IsNotFound(err):
			// object no longer exists, as desired
		case errors.IsConflict(err):
			// delete failed because our UID precondition conflicted
			// this could mean another object exists with a different UID, re-run
			return o.run(attempts + 1)
		default:
			// return other errors
			return nil, err
		}
		// continue to create
		fallthrough
	case ReconcileCreate:
		created, err := o.Client.Create(result.RoleBinding)
		// If created since we started this reconcile, re-run
		if errors.IsAlreadyExists(err) {
			return o.run(attempts + 1)
		}
		if err != nil {
			return nil, err
		}
		result.RoleBinding = created

	case ReconcileUpdate:
		updated, err := o.Client.Update(result.RoleBinding)
		// If deleted since we started this reconcile, re-run
		if errors.IsNotFound(err) {
			return o.run(attempts + 1)
		}
		if err != nil {
			return nil, err
		}
		result.RoleBinding = updated

	case ReconcileNone:
		// no-op

	default:
		return nil, fmt.Errorf("invalid operation: %v", result.Operation)
	}

	return result, nil
}


// computeReconciledRoleBinding returns the rolebinding that must be created and/or updated to make the
// existing rolebinding's subjects, roleref, labels, and annotations match the expected rolebinding
func computeReconciledRoleBinding(existing, expected RoleBinding, removeExtraSubjects bool) (*ReconcileClusterRoleBindingResult, error) {
	result := &ReconcileClusterRoleBindingResult{Operation: ReconcileNone}

	result.Protected = (existing.GetAnnotations()[rbacv1.AutoUpdateAnnotationKey] == "false")

	// Reset the binding completely if the roleRef is different
	if expected.GetRoleRef() != existing.GetRoleRef() {
		result.RoleBinding = expected
		result.Operation = ReconcileRecreate
		return result, nil
	}

	// Start with a copy of the existing object
	result.RoleBinding = existing.DeepCopyRoleBinding()

	// Merge expected annotations and labels
	result.RoleBinding.SetAnnotations(merge(expected.GetAnnotations(), result.RoleBinding.GetAnnotations()))
	if !reflect.DeepEqual(result.RoleBinding.GetAnnotations(), existing.GetAnnotations()) {
		result.Operation = ReconcileUpdate
	}
	result.RoleBinding.SetLabels(merge(expected.GetLabels(), result.RoleBinding.GetLabels()))
	if !reflect.DeepEqual(result.RoleBinding.GetLabels(), existing.GetLabels()) {
		result.Operation = ReconcileUpdate
	}

	// Compute extra and missing subjects
	result.MissingSubjects, result.ExtraSubjects = diffSubjectLists(expected.GetSubjects(), existing.GetSubjects())

	switch {
	case !removeExtraSubjects && len(result.MissingSubjects) > 0:
		// add missing subjects in the union case
		result.RoleBinding.SetSubjects(append(result.RoleBinding.GetSubjects(), result.MissingSubjects...))
		result.Operation = ReconcileUpdate

	case removeExtraSubjects && (len(result.MissingSubjects) > 0 || len(result.ExtraSubjects) > 0):
		// stomp to expected subjects in the non-union case
		result.RoleBinding.SetSubjects(expected.GetSubjects())
		result.Operation = ReconcileUpdate
	}

	return result, nil
}


func contains(list []rbacv1.Subject, item rbacv1.Subject) bool {
	for _, listItem := range list {
		if listItem == item {
			return true
		}
	}
	return false
}


// diffSubjectLists returns lists containing the items unique to each provided list:
//
//	list1Only = list1 - list2
//	list2Only = list2 - list1
//
// if both returned lists are empty, the provided lists are equal
func diffSubjectLists(list1 []rbacv1.Subject, list2 []rbacv1.Subject) (list1Only []rbacv1.Subject, list2Only []rbacv1.Subject) {
	for _, list1Item := range list1 {
		if !contains(list2, list1Item) {
			if !contains(list1Only, list1Item) {
				list1Only = append(list1Only, list1Item)
			}
		}
	}
	for _, list2Item := range list2 {
		if !contains(list1, list2Item) {
			if !contains(list2Only, list2Item) {
				list2Only = append(list2Only, list2Item)
			}
		}
	}
	return
}


func (o RoleRuleOwner) GetObject() runtime.Object {
	return o.Role
}


func (o RoleRuleOwner) GetNamespace() string {
	return o.Role.Namespace
}


func (o RoleRuleOwner) GetName() string

func (o RoleRuleOwner) GetName() string {
	return o.Role.Name
}


func (o RoleRuleOwner) GetLabels() map[string]string {
	return o.Role.Labels
}


func (o RoleRuleOwner) SetLabels(in map[string]string) {
	o.Role.Labels = in
}


func (o RoleRuleOwner) GetAnnotations() map[string]string {
	return o.Role.Annotations
}


func (o RoleRuleOwner) SetAnnotations(in map[string]string) {
	o.Role.Annotations = in
}


func (o RoleRuleOwner) GetRules() []rbacv1.PolicyRule {
	return o.Role.Rules
}


func (o RoleRuleOwner) SetRules(in []rbacv1.PolicyRule) {
	o.Role.Rules = in
}


func (o RoleRuleOwner) GetAggregationRule() *rbacv1.AggregationRule {
	return nil
}


func (o RoleRuleOwner) SetAggregationRule(in *rbacv1.AggregationRule) {
}


func (c RoleModifier) Get(namespace, name string) (RuleOwner, error) {
	ret, err := c.Client.Roles(namespace).Get(context.TODO(), name, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}
	return RoleRuleOwner{Role: ret}, err
}


func (c RoleModifier) Create(in RuleOwner) (RuleOwner, error) {
	if err := tryEnsureNamespace(c.NamespaceClient, in.GetNamespace()); err != nil {
		return nil, err
	}

	ret, err := c.Client.Roles(in.GetNamespace()).Create(context.TODO(), in.(RoleRuleOwner).Role, metav1.CreateOptions{})
	if err != nil {
		return nil, err
	}
	return RoleRuleOwner{Role: ret}, err
}


func (c RoleModifier) Update(in RuleOwner) (RuleOwner, error) {
	ret, err := c.Client.Roles(in.GetNamespace()).Update(context.TODO(), in.(RoleRuleOwner).Role, metav1.UpdateOptions{})
	if err != nil {
		return nil, err
	}
	return RoleRuleOwner{Role: ret}, err

}


func (o RoleBindingAdapter) GetObject() runtime.Object {
	return o.RoleBinding
}


func (o RoleBindingAdapter) GetNamespace() string {
	return o.RoleBinding.Namespace
}


func (o RoleBindingAdapter) GetName() string {
	return o.RoleBinding.Name
}


func (o RoleBindingAdapter) GetUID() types.UID {
	return o.RoleBinding.UID
}


func (o RoleBindingAdapter) GetLabels() map[string]string {
	return o.RoleBinding.Labels
}


func (o RoleBindingAdapter) SetLabels(in map[string]string) {
	o.RoleBinding.Labels = in
}


func (o RoleBindingAdapter) GetAnnotations() map[string]string {
	return o.RoleBinding.Annotations
}


func (o RoleBindingAdapter) SetAnnotations(in map[string]string) {
	o.RoleBinding.Annotations = in
}


func (o RoleBindingAdapter) GetRoleRef() rbacv1.RoleRef {
	return o.RoleBinding.RoleRef
}


func (o RoleBindingAdapter) GetSubjects() []rbacv1.Subject {
	return o.RoleBinding.Subjects
}


func (o RoleBindingAdapter) SetSubjects(in []rbacv1.Subject) {
	o.RoleBinding.Subjects = in
}


func (c RoleBindingClientAdapter) Get(namespace, name string) (RoleBinding, error) {
	ret, err := c.Client.RoleBindings(namespace).Get(context.TODO(), name, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}
	return RoleBindingAdapter{RoleBinding: ret}, err
}


func (c RoleBindingClientAdapter) Create(in RoleBinding) (RoleBinding, error) {
	if err := tryEnsureNamespace(c.NamespaceClient, in.GetNamespace()); err != nil {
		return nil, err
	}

	ret, err := c.Client.RoleBindings(in.GetNamespace()).Create(context.TODO(), in.(RoleBindingAdapter).RoleBinding, metav1.CreateOptions{})
	if err != nil {
		return nil, err
	}
	return RoleBindingAdapter{RoleBinding: ret}, err
}


func (c RoleBindingClientAdapter) Update(in RoleBinding) (RoleBinding, error) {
	ret, err := c.Client.RoleBindings(in.GetNamespace()).Update(context.TODO(), in.(RoleBindingAdapter).RoleBinding, metav1.UpdateOptions{})
	if err != nil {
		return nil, err
	}
	return RoleBindingAdapter{RoleBinding: ret}, err

}


func (c RoleBindingClientAdapter) Delete(namespace, name string, uid types.UID) error {
	return c.Client.RoleBindings(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}})
}


// Covers determines whether or not the ownerRules cover the servantRules in terms of allowed actions.
// It returns whether or not the ownerRules cover and a list of the rules that the ownerRules do not cover.
func Covers(ownerRules, servantRules []rbacv1.PolicyRule) (bool, []rbacv1.PolicyRule) {
	// 1.  Break every servantRule into individual rule tuples: group, verb, resource, resourceName
	// 2.  Compare the mini-rules against each owner rule.  Because the breakdown is down to the most atomic level, we're guaranteed that each mini-servant rule will be either fully covered or not covered by a single owner rule
	// 3.  Any left over mini-rules means that we are not covered and we have a nice list of them.
	// TODO: it might be nice to collapse the list down into something more human readable

	subrules := []rbacv1.PolicyRule{}
	for _, servantRule := range servantRules {
		subrules = append(subrules, BreakdownRule(servantRule)...)
	}

	uncoveredRules := []rbacv1.PolicyRule{}
	for _, subrule := range subrules {
		covered := false
		for _, ownerRule := range ownerRules {
			if ruleCovers(ownerRule, subrule) {
				covered = true
				break
			}
		}

		if !covered {
			uncoveredRules = append(uncoveredRules, subrule)
		}
	}

	return (len(uncoveredRules) == 0), uncoveredRules
}


// BreadownRule takes a rule and builds an equivalent list of rules that each have at most one verb, one
// resource, and one resource name
func BreakdownRule(rule rbacv1.PolicyRule) []rbacv1.PolicyRule {
	subrules := []rbacv1.PolicyRule{}
	for _, group := range rule.APIGroups {
		for _, resource := range rule.Resources {
			for _, verb := range rule.Verbs {
				if len(rule.ResourceNames) > 0 {
					for _, resourceName := range rule.ResourceNames {
						subrules = append(subrules, rbacv1.PolicyRule{APIGroups: []string{group}, Resources: []string{resource}, Verbs: []string{verb}, ResourceNames: []string{resourceName}})
					}

				} else {
					subrules = append(subrules, rbacv1.PolicyRule{APIGroups: []string{group}, Resources: []string{resource}, Verbs: []string{verb}})
				}

			}
		}
	}

	// Non-resource URLs are unique because they only combine with verbs.
	for _, nonResourceURL := range rule.NonResourceURLs {
		for _, verb := range rule.Verbs {
			subrules = append(subrules, rbacv1.PolicyRule{NonResourceURLs: []string{nonResourceURL}, Verbs: []string{verb}})
		}
	}

	return subrules
}


func has(set []string, ele string) bool

func has(set []string, ele string) bool {
	for _, s := range set {
		if s == ele {
			return true
		}
	}
	return false
}


func hasAll(set, contains []string) bool {
	owning := make(map[string]struct{}, len(set))
	for _, ele := range set {
		owning[ele] = struct{}{}
	}
	for _, ele := range contains {
		if _, ok := owning[ele]; !ok {
			return false
		}
	}
	return true
}


func resourceCoversAll(setResources, coversResources []string) bool {
	// if we have a star or an exact match on all resources, then we match
	if has(setResources, rbacv1.ResourceAll) || hasAll(setResources, coversResources) {
		return true
	}

	for _, path := range coversResources {
		// if we have an exact match, then we match.
		if has(setResources, path) {
			continue
		}
		// if we're not a subresource, then we definitely don't match.  fail.
		if !strings.Contains(path, "/") {
			return false
		}
		tokens := strings.SplitN(path, "/", 2)
		resourceToCheck := "*/" + tokens[1]
		if !has(setResources, resourceToCheck) {
			return false
		}
	}

	return true
}


func nonResourceURLsCoversAll(set, covers []string) bool {
	for _, path := range covers {
		covered := false
		for _, owner := range set {
			if nonResourceURLCovers(owner, path) {
				covered = true
				break
			}
		}
		if !covered {
			return false
		}
	}
	return true
}


func nonResourceURLCovers(ownerPath, subPath string) bool {
	if ownerPath == subPath {
		return true
	}
	return strings.HasSuffix(ownerPath, "*") && strings.HasPrefix(subPath, strings.TrimRight(ownerPath, "*"))
}


// ruleCovers determines whether the ownerRule (which may have multiple verbs, resources, and resourceNames) covers
// the subrule (which may only contain at most one verb, resource, and resourceName)
func ruleCovers(ownerRule, subRule rbacv1.PolicyRule) bool {
	verbMatches := has(ownerRule.Verbs, rbacv1.VerbAll) || hasAll(ownerRule.Verbs, subRule.Verbs)
	groupMatches := has(ownerRule.APIGroups, rbacv1.APIGroupAll) || hasAll(ownerRule.APIGroups, subRule.APIGroups)
	resourceMatches := resourceCoversAll(ownerRule.Resources, subRule.Resources)
	nonResourceURLMatches := nonResourceURLsCoversAll(ownerRule.NonResourceURLs, subRule.NonResourceURLs)

	resourceNameMatches := false

	if len(subRule.ResourceNames) == 0 {
		resourceNameMatches = (len(ownerRule.ResourceNames) == 0)
	} else {
		resourceNameMatches = (len(ownerRule.ResourceNames) == 0) || hasAll(ownerRule.ResourceNames, subRule.ResourceNames)
	}

	return verbMatches && groupMatches && resourceMatches && resourceNameMatches && nonResourceURLMatches
}


func GetZoneKey(node *v1.Node) string

// GetZoneKey is a helper function that builds a string identifier that is unique per failure-zone;
// it returns empty-string for no zone.
// Since there are currently two separate zone keys:
//   - "failure-domain.beta.kubernetes.io/zone"
//   - "topology.kubernetes.io/zone"
//
// GetZoneKey will first check failure-domain.beta.kubernetes.io/zone and if not exists, will then check
// topology.kubernetes.io/zone
func GetZoneKey(node *v1.Node) string {
	labels := node.Labels
	if labels == nil {
		return ""
	}

	// TODO: "failure-domain.beta..." names are deprecated, but will
	// stick around a long time due to existing on old extant objects like PVs.
	// Maybe one day we can stop considering them (see #88493).
	zone, ok := labels[v1.LabelFailureDomainBetaZone]
	if !ok {
		zone, _ = labels[v1.LabelTopologyZone]
	}

	region, ok := labels[v1.LabelFailureDomainBetaRegion]
	if !ok {
		region, _ = labels[v1.LabelTopologyRegion]
	}

	if region == "" && zone == "" {
		return ""
	}

	// We include the null character just in case region or failureDomain has a colon
	// (We do assume there's no null characters in a region or failureDomain)
	// As a nice side-benefit, the null character is not printed by fmt.Print or glog
	return region + ":\x00:" + zone
}


// PatchNodeCIDRs patches the specified node.CIDR=cidrs[0] and node.CIDRs to the given value.
func PatchNodeCIDRs(c clientset.Interface, node types.NodeName, cidrs []string) error {
	// set the pod cidrs list and set the old pod cidr field
	patch := nodeForCIDRMergePatch{
		Spec: nodeSpecForMergePatch{
			PodCIDR:  cidrs[0],
			PodCIDRs: cidrs,
		},
	}

	patchBytes, err := json.Marshal(&patch)
	if err != nil {
		return fmt.Errorf("failed to json.Marshal CIDR: %v", err)
	}
	klog.V(4).Infof("cidrs patch bytes are:%s", string(patchBytes))
	if _, err := c.CoreV1().Nodes().Patch(context.TODO(), string(node), types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}); err != nil {
		return fmt.Errorf("failed to patch node CIDR: %v", err)
	}
	return nil
}


// GetNodeCondition extracts the provided condition from the given status and returns that.
// Returns nil and -1 if the condition is not present, and the index of the located condition.
func GetNodeCondition(status *v1.NodeStatus, conditionType v1.NodeConditionType) (int, *v1.NodeCondition) {
	if status == nil {
		return -1, nil
	}
	for i := range status.Conditions {
		if status.Conditions[i].Type == conditionType {
			return i, &status.Conditions[i]
		}
	}
	return -1, nil
}


// SetNodeCondition updates specific node condition with patch operation.
func SetNodeCondition(c clientset.Interface, node types.NodeName, condition v1.NodeCondition) error {
	condition.LastHeartbeatTime = metav1.NewTime(time.Now())
	patch, err := json.Marshal(map[string]interface{}{
		"status": map[string]interface{}{
			"conditions": []v1.NodeCondition{condition},
		},
	})
	if err != nil {
		return err
	}
	_, err = c.CoreV1().Nodes().PatchStatus(context.TODO(), string(node), patch)
	return err
}


// GetHostname returns OS's hostname if 'hostnameOverride' is empty; otherwise, it returns
// 'hostnameOverride'. In either case, the value is canonicalized (trimmed and
// lowercased).
func GetHostname(hostnameOverride string) (string, error) {
	hostName := hostnameOverride
	if len(hostName) == 0 {
		nodeName, err := os.Hostname()
		if err != nil {
			return "", fmt.Errorf("couldn't determine hostname: %w", err)
		}
		hostName = nodeName
	}

	// Trim whitespaces first to avoid getting an empty hostname
	// For linux, the hostname is read from file /proc/sys/kernel/hostname directly
	hostName = strings.TrimSpace(hostName)
	if len(hostName) == 0 {
		return "", fmt.Errorf("empty hostname is invalid")
	}

	return strings.ToLower(hostName), nil
}


// parseNodeIP implements ParseNodeIPArgument and ParseNodeIPAnnotation
func parseNodeIP(nodeIP string, allowDual, sloppy bool) ([]net.IP, error) {
	var nodeIPs []net.IP
	if nodeIP != "" || !sloppy {
		for _, ip := range strings.Split(nodeIP, ",") {
			if sloppy {
				ip = strings.TrimSpace(ip)
			}
			parsedNodeIP := netutils.ParseIPSloppy(ip)
			if parsedNodeIP == nil {
				if sloppy {
					klog.InfoS("Could not parse node IP. Ignoring", "IP", ip)
				} else {
					return nil, fmt.Errorf("could not parse %q", ip)
				}
			} else {
				nodeIPs = append(nodeIPs, parsedNodeIP)
			}
		}
	}

	if len(nodeIPs) > 2 || (len(nodeIPs) == 2 && netutils.IsIPv6(nodeIPs[0]) == netutils.IsIPv6(nodeIPs[1])) {
		return nil, fmt.Errorf("must contain either a single IP or a dual-stack pair of IPs")
	} else if len(nodeIPs) == 2 && !allowDual {
		return nil, fmt.Errorf("dual-stack not supported in this configuration")
	} else if len(nodeIPs) == 2 && (nodeIPs[0].IsUnspecified() || nodeIPs[1].IsUnspecified()) {
		return nil, fmt.Errorf("dual-stack node IP cannot include '0.0.0.0' or '::'")
	}

	return nodeIPs, nil
}


// ParseNodeIPArgument parses kubelet's --node-ip argument. If nodeIP contains invalid
// values, they will be logged and ignored. Dual-stack node IPs are allowed if
// cloudProvider is unset, or if it is `"external"` and allowCloudDualStack is true.
func ParseNodeIPArgument(nodeIP, cloudProvider string, allowCloudDualStack bool) ([]net.IP, error) {
	var allowDualStack bool
	if (cloudProvider == cloudProviderNone) || (cloudProvider == cloudProviderExternal && allowCloudDualStack) {
		allowDualStack = true
	}
	return parseNodeIP(nodeIP, allowDualStack, true)
}


// ParseNodeIPAnnotation parses the `alpha.kubernetes.io/provided-node-ip` annotation,
// which can be either a single IP address or (if allowDualStack is true) a
// comma-separated pair of IP addresses. Unlike with ParseNodeIPArgument, invalid values
// are considered an error.
func ParseNodeIPAnnotation(nodeIP string, allowDualStack bool) ([]net.IP, error) {
	return parseNodeIP(nodeIP, allowDualStack, false)
}


// PatchNodeStatus patches node status.
func PatchNodeStatus(c v1core.CoreV1Interface, nodeName types.NodeName, oldNode *v1.Node, newNode *v1.Node) (*v1.Node, []byte, error) {
	patchBytes, err := preparePatchBytesforNodeStatus(nodeName, oldNode, newNode)
	if err != nil {
		return nil, nil, err
	}

	updatedNode, err := c.Nodes().Patch(context.TODO(), string(nodeName), types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status")
	if err != nil {
		return nil, nil, fmt.Errorf("failed to patch status %q for node %q: %v", patchBytes, nodeName, err)
	}
	return updatedNode, patchBytes, nil
}


func preparePatchBytesforNodeStatus(nodeName types.NodeName, oldNode *v1.Node, newNode *v1.Node) ([]byte, error) {
	oldData, err := json.Marshal(oldNode)
	if err != nil {
		return nil, fmt.Errorf("failed to Marshal oldData for node %q: %v", nodeName, err)
	}

	// NodeStatus.Addresses is incorrectly annotated as patchStrategy=merge, which
	// will cause strategicpatch.CreateTwoWayMergePatch to create an incorrect patch
	// if it changed.
	manuallyPatchAddresses := (len(oldNode.Status.Addresses) > 0) && !equality.Semantic.DeepEqual(oldNode.Status.Addresses, newNode.Status.Addresses)

	// Reset spec to make sure only patch for Status or ObjectMeta is generated.
	// Note that we don't reset ObjectMeta here, because:
	// 1. This aligns with Nodes().UpdateStatus().
	// 2. Some component does use this to update node annotations.
	diffNode := newNode.DeepCopy()
	diffNode.Spec = oldNode.Spec
	if manuallyPatchAddresses {
		diffNode.Status.Addresses = oldNode.Status.Addresses
	}
	newData, err := json.Marshal(diffNode)
	if err != nil {
		return nil, fmt.Errorf("failed to Marshal newData for node %q: %v", nodeName, err)
	}

	patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{})
	if err != nil {
		return nil, fmt.Errorf("failed to CreateTwoWayMergePatch for node %q: %v", nodeName, err)
	}
	if manuallyPatchAddresses {
		patchBytes, err = fixupPatchForNodeStatusAddresses(patchBytes, newNode.Status.Addresses)
		if err != nil {
			return nil, fmt.Errorf("failed to fix up NodeAddresses in patch for node %q: %v", nodeName, err)
		}
	}

	return patchBytes, nil
}


// fixupPatchForNodeStatusAddresses adds a replace-strategy patch for Status.Addresses to
// the existing patch
func fixupPatchForNodeStatusAddresses(patchBytes []byte, addresses []v1.NodeAddress) ([]byte, error) {
	// Given patchBytes='{"status": {"conditions": [ ... ], "phase": ...}}' and
	// addresses=[{"type": "InternalIP", "address": "10.0.0.1"}], we need to generate:
	//
	//   {
	//     "status": {
	//       "conditions": [ ... ],
	//       "phase": ...,
	//       "addresses": [
	//         {
	//           "type": "InternalIP",
	//           "address": "10.0.0.1"
	//         },
	//         {
	//           "$patch": "replace"
	//         }
	//       ]
	//     }
	//   }

	var patchMap map[string]interface{}
	if err := json.Unmarshal(patchBytes, &patchMap); err != nil {
		return nil, err
	}

	addrBytes, err := json.Marshal(addresses)
	if err != nil {
		return nil, err
	}
	var addrArray []interface{}
	if err := json.Unmarshal(addrBytes, &addrArray); err != nil {
		return nil, err
	}
	addrArray = append(addrArray, map[string]interface{}{"$patch": "replace"})

	status := patchMap["status"]
	if status == nil {
		status = map[string]interface{}{}
		patchMap["status"] = status
	}
	statusMap, ok := status.(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("unexpected data in patch")
	}
	statusMap["addresses"] = addrArray

	return json.Marshal(patchMap)
}


func New() Interface

// New returns a new Interface for accessing sysctl
func New() Interface {
	return &procSysctl{}
}


// GetSysctl returns the value for the specified sysctl setting
func (*procSysctl) GetSysctl(sysctl string) (int, error) {
	data, err := os.ReadFile(path.Join(sysctlBase, sysctl))
	if err != nil {
		return -1, err
	}
	val, err := strconv.Atoi(strings.Trim(string(data), " \n"))
	if err != nil {
		return -1, err
	}
	return val, nil
}


// SetSysctl modifies the specified sysctl flag to the new value
func (*procSysctl) SetSysctl(sysctl string, newVal int) error {
	return os.WriteFile(path.Join(sysctlBase, sysctl), []byte(strconv.Itoa(newVal)), 0640)
}


func PodPriority(pod *v1.Pod) int32

// PodPriority returns priority of the given pod.
func PodPriority(pod *v1.Pod) int32 {
	if pod.Spec.Priority != nil {
		return *pod.Spec.Priority
	}
	// When priority of a running pod is nil, it means it was created at a time
	// that there was no global default priority class and the priority class
	// name of the pod was empty. So, we resolve to the static default priority.
	return 0
}


func MatchNodeSelectorTerms(

// MatchNodeSelectorTerms checks whether the node labels and fields match node selector terms in ORed;
// nil or empty term matches no objects.
func MatchNodeSelectorTerms(
	node *v1.Node,
	nodeSelector *v1.NodeSelector,
) (bool, error) {
	if node == nil {
		return false, nil
	}
	return nodeaffinity.NewLazyErrorNodeSelector(nodeSelector).Match(node)
}


// GetAvoidPodsFromNodeAnnotations scans the list of annotations and
// returns the pods that needs to be avoided for this node from scheduling
func GetAvoidPodsFromNodeAnnotations(annotations map[string]string) (v1.AvoidPods, error) {
	var avoidPods v1.AvoidPods
	if len(annotations) > 0 && annotations[v1.PreferAvoidPodsAnnotationKey] != "" {
		err := json.Unmarshal([]byte(annotations[v1.PreferAvoidPodsAnnotationKey]), &avoidPods)
		if err != nil {
			return avoidPods, err
		}
	}
	return avoidPods, nil
}


// TolerationsTolerateTaint checks if taint is tolerated by any of the tolerations.
func TolerationsTolerateTaint(tolerations []v1.Toleration, taint *v1.Taint) bool {
	for i := range tolerations {
		if tolerations[i].ToleratesTaint(taint) {
			return true
		}
	}
	return false
}


// FindMatchingUntoleratedTaint checks if the given tolerations tolerates
// all the filtered taints, and returns the first taint without a toleration
// Returns true if there is an untolerated taint
// Returns false if all taints are tolerated
func FindMatchingUntoleratedTaint(taints []v1.Taint, tolerations []v1.Toleration, inclusionFilter taintsFilterFunc) (v1.Taint, bool) {
	filteredTaints := getFilteredTaints(taints, inclusionFilter)
	for _, taint := range filteredTaints {
		if !TolerationsTolerateTaint(tolerations, &taint) {
			return taint, true
		}
	}
	return v1.Taint{}, false
}


// getFilteredTaints returns a list of taints satisfying the filter predicate
func getFilteredTaints(taints []v1.Taint, inclusionFilter taintsFilterFunc) []v1.Taint {
	if inclusionFilter == nil {
		return taints
	}
	filteredTaints := []v1.Taint{}
	for _, taint := range taints {
		if !inclusionFilter(&taint) {
			continue
		}
		filteredTaints = append(filteredTaints, taint)
	}
	return filteredTaints
}


// NewNodeSelector returns a NodeSelector or aggregate parsing errors found.
func NewNodeSelector(ns *v1.NodeSelector, opts ...field.PathOption) (*NodeSelector, error) {
	lazy := NewLazyErrorNodeSelector(ns, opts...)
	var errs []error
	for _, term := range lazy.terms {
		if len(term.parseErrs) > 0 {
			errs = append(errs, term.parseErrs...)
		}
	}
	if len(errs) != 0 {
		return nil, errors.Flatten(errors.NewAggregate(errs))
	}
	return &NodeSelector{lazy: *lazy}, nil
}


// NewLazyErrorNodeSelector creates a NodeSelector that only reports parse
// errors when no terms match.
func NewLazyErrorNodeSelector(ns *v1.NodeSelector, opts ...field.PathOption) *LazyErrorNodeSelector {
	p := field.ToPath(opts...)
	parsedTerms := make([]nodeSelectorTerm, 0, len(ns.NodeSelectorTerms))
	path := p.Child("nodeSelectorTerms")
	for i, term := range ns.NodeSelectorTerms {
		// nil or empty term selects no objects
		if isEmptyNodeSelectorTerm(&term) {
			continue
		}
		p := path.Index(i)
		parsedTerms = append(parsedTerms, newNodeSelectorTerm(&term, p))
	}
	return &LazyErrorNodeSelector{
		terms: parsedTerms,
	}
}


// Match checks whether the node labels and fields match the selector terms, ORed;
// nil or empty term matches no objects.
func (ns *NodeSelector) Match(node *v1.Node) bool {
	// parse errors are reported in NewNodeSelector.
	match, _ := ns.lazy.Match(node)
	return match
}


// Match checks whether the node labels and fields match the selector terms, ORed;
// nil or empty term matches no objects.
// Parse errors are only returned if no terms matched.
func (ns *LazyErrorNodeSelector) Match(node *v1.Node) (bool, error) {
	if node == nil {
		return false, nil
	}
	nodeLabels := labels.Set(node.Labels)
	nodeFields := extractNodeFields(node)

	var errs []error
	for _, term := range ns.terms {
		match, tErrs := term.match(nodeLabels, nodeFields)
		if len(tErrs) > 0 {
			errs = append(errs, tErrs...)
			continue
		}
		if match {
			return true, nil
		}
	}
	return false, errors.Flatten(errors.NewAggregate(errs))
}


// NewPreferredSchedulingTerms returns a PreferredSchedulingTerms or all the parsing errors found.
// If a v1.PreferredSchedulingTerm has a 0 weight, its parsing is skipped.
func NewPreferredSchedulingTerms(terms []v1.PreferredSchedulingTerm, opts ...field.PathOption) (*PreferredSchedulingTerms, error) {
	p := field.ToPath(opts...)
	var errs []error
	parsedTerms := make([]preferredSchedulingTerm, 0, len(terms))
	for i, term := range terms {
		path := p.Index(i)
		if term.Weight == 0 || isEmptyNodeSelectorTerm(&term.Preference) {
			continue
		}
		parsedTerm := preferredSchedulingTerm{
			nodeSelectorTerm: newNodeSelectorTerm(&term.Preference, path),
			weight:           int(term.Weight),
		}
		if len(parsedTerm.parseErrs) > 0 {
			errs = append(errs, parsedTerm.parseErrs...)
		} else {
			parsedTerms = append(parsedTerms, parsedTerm)
		}
	}
	if len(errs) != 0 {
		return nil, errors.Flatten(errors.NewAggregate(errs))
	}
	return &PreferredSchedulingTerms{terms: parsedTerms}, nil
}


// Score returns a score for a Node: the sum of the weights of the terms that
// match the Node.
func (t *PreferredSchedulingTerms) Score(node *v1.Node) int64 {
	var score int64
	nodeLabels := labels.Set(node.Labels)
	nodeFields := extractNodeFields(node)
	for _, term := range t.terms {
		// parse errors are reported in NewPreferredSchedulingTerms.
		if ok, _ := term.match(nodeLabels, nodeFields); ok {
			score += int64(term.weight)
		}
	}
	return score
}


func isEmptyNodeSelectorTerm(term *v1.NodeSelectorTerm) bool {
	return len(term.MatchExpressions) == 0 && len(term.MatchFields) == 0
}


func extractNodeFields(n *v1.Node) fields.Set {
	f := make(fields.Set)
	if len(n.Name) > 0 {
		f["metadata.name"] = n.Name
	}
	return f
}


func newNodeSelectorTerm(term *v1.NodeSelectorTerm, path *field.Path) nodeSelectorTerm {
	var parsedTerm nodeSelectorTerm
	var errs []error
	if len(term.MatchExpressions) != 0 {
		p := path.Child("matchExpressions")
		parsedTerm.matchLabels, errs = nodeSelectorRequirementsAsSelector(term.MatchExpressions, p)
		if errs != nil {
			parsedTerm.parseErrs = append(parsedTerm.parseErrs, errs...)
		}
	}
	if len(term.MatchFields) != 0 {
		p := path.Child("matchFields")
		parsedTerm.matchFields, errs = nodeSelectorRequirementsAsFieldSelector(term.MatchFields, p)
		if errs != nil {
			parsedTerm.parseErrs = append(parsedTerm.parseErrs, errs...)
		}
	}
	return parsedTerm
}


func (t *nodeSelectorTerm) match(nodeLabels labels.Set, nodeFields fields.Set) (bool, []error) {
	if t.parseErrs != nil {
		return false, t.parseErrs
	}
	if t.matchLabels != nil && !t.matchLabels.Matches(nodeLabels) {
		return false, nil
	}
	if t.matchFields != nil && len(nodeFields) > 0 && !t.matchFields.Matches(nodeFields) {
		return false, nil
	}
	return true, nil
}


// nodeSelectorRequirementsAsSelector converts the []NodeSelectorRequirement api type into a struct that implements
// labels.Selector.
func nodeSelectorRequirementsAsSelector(nsm []v1.NodeSelectorRequirement, path *field.Path) (labels.Selector, []error) {
	if len(nsm) == 0 {
		return labels.Nothing(), nil
	}
	var errs []error
	selector := labels.NewSelector()
	for i, expr := range nsm {
		p := path.Index(i)
		var op selection.Operator
		switch expr.Operator {
		case v1.NodeSelectorOpIn:
			op = selection.In
		case v1.NodeSelectorOpNotIn:
			op = selection.NotIn
		case v1.NodeSelectorOpExists:
			op = selection.Exists
		case v1.NodeSelectorOpDoesNotExist:
			op = selection.DoesNotExist
		case v1.NodeSelectorOpGt:
			op = selection.GreaterThan
		case v1.NodeSelectorOpLt:
			op = selection.LessThan
		default:
			errs = append(errs, field.NotSupported(p.Child("operator"), expr.Operator, nil))
			continue
		}
		r, err := labels.NewRequirement(expr.Key, op, expr.Values, field.WithPath(p))
		if err != nil {
			errs = append(errs, err)
		} else {
			selector = selector.Add(*r)
		}
	}
	if len(errs) != 0 {
		return nil, errs
	}
	return selector, nil
}


// nodeSelectorRequirementsAsFieldSelector converts the []NodeSelectorRequirement core type into a struct that implements
// fields.Selector.
func nodeSelectorRequirementsAsFieldSelector(nsr []v1.NodeSelectorRequirement, path *field.Path) (fields.Selector, []error) {
	if len(nsr) == 0 {
		return fields.Nothing(), nil
	}
	var errs []error

	var selectors []fields.Selector
	for i, expr := range nsr {
		p := path.Index(i)
		switch expr.Operator {
		case v1.NodeSelectorOpIn:
			if len(expr.Values) != 1 {
				errs = append(errs, field.Invalid(p.Child("values"), expr.Values, "must have one element"))
			} else {
				selectors = append(selectors, fields.OneTermEqualSelector(expr.Key, expr.Values[0]))
			}

		case v1.NodeSelectorOpNotIn:
			if len(expr.Values) != 1 {
				errs = append(errs, field.Invalid(p.Child("values"), expr.Values, "must have one element"))
			} else {
				selectors = append(selectors, fields.OneTermNotEqualSelector(expr.Key, expr.Values[0]))
			}

		default:
			errs = append(errs, field.NotSupported(p.Child("operator"), expr.Operator, validFieldSelectorOperators))
		}
	}

	if len(errs) != 0 {
		return nil, errs
	}
	return fields.AndSelectors(selectors...), nil
}


// GetRequiredNodeAffinity returns the parsing result of pod's nodeSelector and nodeAffinity.
func GetRequiredNodeAffinity(pod *v1.Pod) RequiredNodeAffinity {
	var selector labels.Selector
	if len(pod.Spec.NodeSelector) > 0 {
		selector = labels.SelectorFromSet(pod.Spec.NodeSelector)
	}
	// Use LazyErrorNodeSelector for backwards compatibility of parsing errors.
	var affinity *LazyErrorNodeSelector
	if pod.Spec.Affinity != nil &&
		pod.Spec.Affinity.NodeAffinity != nil &&
		pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil {
		affinity = NewLazyErrorNodeSelector(pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution)
	}
	return RequiredNodeAffinity{labelSelector: selector, nodeSelector: affinity}
}


// Match checks whether the pod is schedulable onto nodes according to
// the requirements in both nodeSelector and nodeAffinity.
func (s RequiredNodeAffinity) Match(node *v1.Node) (bool, error) {
	if s.labelSelector != nil {
		if !s.labelSelector.Matches(labels.Set(node.Labels)) {
			return false, nil
		}
	}
	if s.nodeSelector != nil {
		return s.nodeSelector.Match(node)
	}
	return true, nil
}


// VolumeClaimName returns the name of the PersistentVolumeClaim
// object that gets created for the generic ephemeral inline volume. The
// name is deterministic and therefore this function does not need any
// additional information besides the Pod name and volume name and it
// will never fail.
//
// Before using the PVC for the Pod, the caller must check that it is
// indeed the PVC that was created for the Pod by calling IsUsable.
func VolumeClaimName(pod *v1.Pod, volume *v1.Volume) string {
	return pod.Name + "-" + volume.Name
}


// VolumeIsForPod checks that the PVC is the ephemeral volume that
// was created for the Pod. It returns an error that is informative
// enough to be returned by the caller without adding further details
// about the Pod or PVC.
func VolumeIsForPod(pod *v1.Pod, pvc *v1.PersistentVolumeClaim) error {
	// Checking the namespaces is just a precaution. The caller should
	// never pass in a PVC that isn't from the same namespace as the
	// Pod.
	if pvc.Namespace != pod.Namespace || !metav1.IsControlledBy(pvc, pod) {
		return fmt.Errorf("PVC %s/%s was not created for pod %s/%s (pod is not owner)", pvc.Namespace, pvc.Name, pod.Namespace, pod.Name)
	}
	return nil
}


// GetPersistentVolumeClaimClass returns StorageClassName. If no storage class was
// requested, it returns "".
func GetPersistentVolumeClaimClass(claim *v1.PersistentVolumeClaim) string {
	// Use beta annotation first
	if class, found := claim.Annotations[v1.BetaStorageClassAnnotation]; found {
		return class
	}

	if claim.Spec.StorageClassName != nil {
		return *claim.Spec.StorageClassName
	}

	return ""
}


// GetPersistentVolumeClass returns StorageClassName.
func GetPersistentVolumeClass(volume *v1.PersistentVolume) string {
	// Use beta annotation first
	if class, found := volume.Annotations[v1.BetaStorageClassAnnotation]; found {
		return class
	}

	return volume.Spec.StorageClassName
}


// CheckNodeAffinity looks at the PV node affinity, and checks if the node has the same corresponding labels
// This ensures that we don't mount a volume that doesn't belong to this node
func CheckNodeAffinity(pv *v1.PersistentVolume, nodeLabels map[string]string) error {
	if pv.Spec.NodeAffinity == nil {
		return nil
	}

	if pv.Spec.NodeAffinity.Required != nil {
		node := &v1.Node{ObjectMeta: metav1.ObjectMeta{Labels: nodeLabels}}
		terms := pv.Spec.NodeAffinity.Required
		if matches, err := corev1.MatchNodeSelectorTerms(node, terms); err != nil {
			return err
		} else if !matches {
			return fmt.Errorf("no matching NodeSelectorTerms")
		}
	}

	return nil
}


// IsDelayBindingProvisioning checks if claim provisioning with selected-node annotation
func IsDelayBindingProvisioning(claim *v1.PersistentVolumeClaim) bool {
	// When feature VolumeScheduling enabled,
	// Scheduler signal to the PV controller to start dynamic
	// provisioning by setting the "AnnSelectedNode" annotation
	// in the PVC
	_, ok := claim.Annotations[AnnSelectedNode]
	return ok
}


// IsDelayBindingMode checks if claim is in delay binding mode.
func IsDelayBindingMode(claim *v1.PersistentVolumeClaim, classLister storagelisters.StorageClassLister) (bool, error) {
	className := GetPersistentVolumeClaimClass(claim)
	if className == "" {
		return false, nil
	}

	class, err := classLister.Get(className)
	if err != nil {
		if apierrors.IsNotFound(err) {
			return false, nil
		}
		return false, err
	}

	if class.VolumeBindingMode == nil {
		return false, fmt.Errorf("VolumeBindingMode not set for StorageClass %q", className)
	}

	return *class.VolumeBindingMode == storage.VolumeBindingWaitForFirstConsumer, nil
}


// GetBindVolumeToClaim returns a new volume which is bound to given claim. In
// addition, it returns a bool which indicates whether we made modification on
// original volume.
func GetBindVolumeToClaim(volume *v1.PersistentVolume, claim *v1.PersistentVolumeClaim) (*v1.PersistentVolume, bool, error) {
	dirty := false

	// Check if the volume was already bound (either by user or by controller)
	shouldSetBoundByController := false
	if !IsVolumeBoundToClaim(volume, claim) {
		shouldSetBoundByController = true
	}

	// The volume from method args can be pointing to watcher cache. We must not
	// modify these, therefore create a copy.
	volumeClone := volume.DeepCopy()

	// Bind the volume to the claim if it is not bound yet
	if volume.Spec.ClaimRef == nil ||
		volume.Spec.ClaimRef.Name != claim.Name ||
		volume.Spec.ClaimRef.Namespace != claim.Namespace ||
		volume.Spec.ClaimRef.UID != claim.UID {

		claimRef, err := reference.GetReference(scheme.Scheme, claim)
		if err != nil {
			return nil, false, fmt.Errorf("unexpected error getting claim reference: %w", err)
		}
		volumeClone.Spec.ClaimRef = claimRef
		dirty = true
	}

	// Set AnnBoundByController if it is not set yet
	if shouldSetBoundByController && !metav1.HasAnnotation(volumeClone.ObjectMeta, AnnBoundByController) {
		metav1.SetMetaDataAnnotation(&volumeClone.ObjectMeta, AnnBoundByController, "yes")
		dirty = true
	}

	return volumeClone, dirty, nil
}


// IsVolumeBoundToClaim returns true, if given volume is pre-bound or bound
// to specific claim. Both claim.Name and claim.Namespace must be equal.
// If claim.UID is present in volume.Spec.ClaimRef, it must be equal too.
func IsVolumeBoundToClaim(volume *v1.PersistentVolume, claim *v1.PersistentVolumeClaim) bool {
	if volume.Spec.ClaimRef == nil {
		return false
	}
	if claim.Name != volume.Spec.ClaimRef.Name || claim.Namespace != volume.Spec.ClaimRef.Namespace {
		return false
	}
	if volume.Spec.ClaimRef.UID != "" && claim.UID != volume.Spec.ClaimRef.UID {
		return false
	}
	return true
}


func FindMatchingVolume(

// FindMatchingVolume goes through the list of volumes to find the best matching volume
// for the claim.
//
// This function is used by both the PV controller and scheduler.
//
// delayBinding is true only in the PV controller path.  When set, prebound PVs are still returned
// as a match for the claim, but unbound PVs are skipped.
//
// node is set only in the scheduler path. When set, the PV node affinity is checked against
// the node's labels.
//
// excludedVolumes is only used in the scheduler path, and is needed for evaluating multiple
// unbound PVCs for a single Pod at one time.  As each PVC finds a matching PV, the chosen
// PV needs to be excluded from future matching.
func FindMatchingVolume(
	claim *v1.PersistentVolumeClaim,
	volumes []*v1.PersistentVolume,
	node *v1.Node,
	excludedVolumes map[string]*v1.PersistentVolume,
	delayBinding bool) (*v1.PersistentVolume, error) {

	var smallestVolume *v1.PersistentVolume
	var smallestVolumeQty resource.Quantity
	requestedQty := claim.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	requestedClass := GetPersistentVolumeClaimClass(claim)

	var selector labels.Selector
	if claim.Spec.Selector != nil {
		internalSelector, err := metav1.LabelSelectorAsSelector(claim.Spec.Selector)
		if err != nil {
			return nil, fmt.Errorf("error creating internal label selector for claim: %v: %v", claimToClaimKey(claim), err)
		}
		selector = internalSelector
	}

	// Go through all available volumes with two goals:
	// - find a volume that is either pre-bound by user or dynamically
	//   provisioned for this claim. Because of this we need to loop through
	//   all volumes.
	// - find the smallest matching one if there is no volume pre-bound to
	//   the claim.
	for _, volume := range volumes {
		if _, ok := excludedVolumes[volume.Name]; ok {
			// Skip volumes in the excluded list
			continue
		}
		if volume.Spec.ClaimRef != nil && !IsVolumeBoundToClaim(volume, claim) {
			continue
		}

		volumeQty := volume.Spec.Capacity[v1.ResourceStorage]
		if volumeQty.Cmp(requestedQty) < 0 {
			continue
		}
		// filter out mismatching volumeModes
		if CheckVolumeModeMismatches(&claim.Spec, &volume.Spec) {
			continue
		}

		// check if PV's DeletionTimeStamp is set, if so, skip this volume.
		if volume.ObjectMeta.DeletionTimestamp != nil {
			continue
		}

		nodeAffinityValid := true
		if node != nil {
			// Scheduler path, check that the PV NodeAffinity
			// is satisfied by the node
			// CheckNodeAffinity is the most expensive call in this loop.
			// We should check cheaper conditions first or consider optimizing this function.
			err := CheckNodeAffinity(volume, node.Labels)
			if err != nil {
				nodeAffinityValid = false
			}
		}

		if IsVolumeBoundToClaim(volume, claim) {
			// If PV node affinity is invalid, return no match.
			// This means the prebound PV (and therefore PVC)
			// is not suitable for this node.
			if !nodeAffinityValid {
				return nil, nil
			}

			return volume, nil
		}

		if node == nil && delayBinding {
			// PV controller does not bind this claim.
			// Scheduler will handle binding unbound volumes
			// Scheduler path will have node != nil
			continue
		}

		// filter out:
		// - volumes in non-available phase
		// - volumes whose labels don't match the claim's selector, if specified
		// - volumes in Class that is not requested
		// - volumes whose NodeAffinity does not match the node
		if volume.Status.Phase != v1.VolumeAvailable {
			// We ignore volumes in non-available phase, because volumes that
			// satisfies matching criteria will be updated to available, binding
			// them now has high chance of encountering unnecessary failures
			// due to API conflicts.
			continue
		} else if selector != nil && !selector.Matches(labels.Set(volume.Labels)) {
			continue
		}
		if GetPersistentVolumeClass(volume) != requestedClass {
			continue
		}
		if !nodeAffinityValid {
			continue
		}

		if node != nil {
			// Scheduler path
			// Check that the access modes match
			if !CheckAccessModes(claim, volume) {
				continue
			}
		}

		if smallestVolume == nil || smallestVolumeQty.Cmp(volumeQty) > 0 {
			smallestVolume = volume
			smallestVolumeQty = volumeQty
		}
	}

	if smallestVolume != nil {
		// Found a matching volume
		return smallestVolume, nil
	}

	return nil, nil
}