Restore

// Restore restores the pool to the previously captured state. ErrMismatchedNetwork
// is returned if the provided IPNet range doesn't exactly match the previous range.
func (r *Range) Restore(net *net.IPNet, data []byte) error {
	if !net.IP.Equal(r.net.IP) || net.Mask.String() != r.net.Mask.String() {
		return ErrMismatchedNetwork
	}
	snapshottable, ok := r.alloc.(allocator.Snapshottable)
	if !ok {
		return fmt.Errorf("not a snapshottable allocator")
	}
	if err := snapshottable.Restore(net.String(), data); err != nil {
		return fmt.Errorf("restoring snapshot encountered %v", err)
	}
	return nil
}


contains

// contains returns true and the offset if the ip is in the range, and false
// and nil otherwise. The first and last addresses of the CIDR are omitted.
func (r *Range) contains(ip net.IP) (bool, int) {
	if !r.net.Contains(ip) {
		return false, 0
	}

	offset := calculateIPOffset(r.base, ip)
	if offset < 0 || offset >= r.max {
		return false, 0
	}
	return true, offset
}


Destroy

// Destroy shuts down internal allocator.
func (r *Range) Destroy() {
	r.alloc.Destroy()
}


EnableMetrics

// EnableMetrics enables metrics recording.
func (r *Range) EnableMetrics() {
	registerMetrics()
	r.metrics = &metricsRecorder{}
}


calculateIPOffset

// calculateIPOffset calculates the integer offset of ip from base such that
// base + offset = ip. It requires ip >= base.
func calculateIPOffset(base *big.Int, ip net.IP) int {
	return int(big.NewInt(0).Sub(netutils.BigForIP(ip), base).Int64())
}


calculateRangeOffset

// calculateRangeOffset estimates the offset used on the range for statically allocation based on
// the following formula `min(max($min, cidrSize/$step), $max)`, described as ~never less than
// $min or more than $max, with a graduated step function between them~. The function returns 0
// if any of the parameters is invalid.
func calculateRangeOffset(cidr *net.IPNet) int {
	// default values for min(max($min, cidrSize/$step), $max)
	const (
		min  = 16
		max  = 256
		step = 16
	)

	cidrSize := netutils.RangeSize(cidr)
	// available addresses are always less than the cidr size
	// A /28 CIDR returns 16 addresses, but 2 of them, the network
	// and broadcast addresses are not available.
	if cidrSize <= min {
		return 0
	}

	offset := cidrSize / step
	if offset < min {
		return min
	}
	if offset > max {
		return max
	}
	return int(offset)
}


Allocate

func (dry dryRunRange) Allocate(ip net.IP) error {
	return dry.real.allocate(ip, dryRunTrue)
}


AllocateNext

func (dry dryRunRange) AllocateNext() (net.IP, error) {
	return dry.real.allocateNext(dryRunTrue)
}


Release

func (dry dryRunRange) Release(ip net.IP) error {
	return dry.real.release(ip, dryRunTrue)
}


ForEach

func (dry dryRunRange) ForEach(cb func(net.IP)) {
	dry.real.ForEach(cb)
}


CIDR

func (dry dryRunRange) CIDR() net.IPNet {
	return dry.real.CIDR()
}


IPFamily

func (dry dryRunRange) IPFamily() api.IPFamily {
	return dry.real.IPFamily()
}


DryRun

func (dry dryRunRange) DryRun() Interface {
	return dry
}


Has

func (dry dryRunRange) Has(ip net.IP) bool {
	return dry.real.Has(ip)
}


EnableMetrics

func (dry dryRunRange) EnableMetrics() {
}


NewRepair

// NewRepair creates a controller that periodically ensures that all clusterIPs are uniquely allocated across the cluster
// and generates informational warnings for a cluster that is not in sync.
func NewRepair(interval time.Duration, serviceClient corev1client.ServicesGetter, eventClient eventsv1client.EventsV1Interface, network *net.IPNet, alloc rangeallocation.RangeRegistry, secondaryNetwork *net.IPNet, secondaryAlloc rangeallocation.RangeRegistry) *Repair {
	eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: eventClient})
	recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, "ipallocator-repair-controller")

	// build *ByFamily struct members
	networkByFamily := make(map[v1.IPFamily]*net.IPNet)
	allocatorByFamily := make(map[v1.IPFamily]rangeallocation.RangeRegistry)
	leaksByFamily := make(map[v1.IPFamily]map[string]int)

	primary := v1.IPv4Protocol
	secondary := v1.IPv6Protocol
	if netutils.IsIPv6(network.IP) {
		primary = v1.IPv6Protocol
	}

	networkByFamily[primary] = network
	allocatorByFamily[primary] = alloc
	leaksByFamily[primary] = make(map[string]int)

	if secondaryNetwork != nil && secondaryNetwork.IP != nil {
		if primary == v1.IPv6Protocol {
			secondary = v1.IPv4Protocol
		}
		networkByFamily[secondary] = secondaryNetwork
		allocatorByFamily[secondary] = secondaryAlloc
		leaksByFamily[secondary] = make(map[string]int)
	}

	return &Repair{
		interval:      interval,
		serviceClient: serviceClient,

		networkByFamily:   networkByFamily,
		allocatorByFamily: allocatorByFamily,

		leaksByFamily: leaksByFamily,
		broadcaster:   eventBroadcaster,
		recorder:      recorder,
	}
}


RunUntil

// RunUntil starts the controller until the provided ch is closed.
func (c *Repair) RunUntil(onFirstSuccess func(), stopCh chan struct{}) {
	c.broadcaster.StartRecordingToSink(stopCh)
	defer c.broadcaster.Shutdown()

	var once sync.Once
	wait.Until(func() {
		if err := c.runOnce(); err != nil {
			runtime.HandleError(err)
			return
		}
		once.Do(onFirstSuccess)
	}, c.interval, stopCh)
}


runOnce

// runOnce verifies the state of the cluster IP allocations and returns an error if an unrecoverable problem occurs.
func (c *Repair) runOnce() error {
	return retry.RetryOnConflict(retry.DefaultBackoff, c.doRunOnce)
}


doRunOnce

// doRunOnce verifies the state of the cluster IP allocations and returns an error if an unrecoverable problem occurs.
func (c *Repair) doRunOnce() error {
	// TODO: (per smarterclayton) if Get() or ListServices() is a weak consistency read,
	// or if they are executed against different leaders,
	// the ordering guarantee required to ensure no IP is allocated twice is violated.
	// ListServices must return a ResourceVersion higher than the etcd index Get triggers,
	// and the release code must not release services that have had IPs allocated but not yet been created
	// See #8295

	// If etcd server is not running we should wait for some time and fail only then. This is particularly
	// important when we start apiserver and etcd at the same time.
	snapshotByFamily := make(map[v1.IPFamily]*api.RangeAllocation)
	storedByFamily := make(map[v1.IPFamily]ipallocator.Interface)

	err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
		for family, allocator := range c.allocatorByFamily {
			// get snapshot if it is not there
			if _, ok := snapshotByFamily[family]; !ok {
				snapshot, err := allocator.Get()
				if err != nil {
					return false, err
				}

				snapshotByFamily[family] = snapshot
			}
		}
		return true, nil
	})

	if err != nil {
		return fmt.Errorf("unable to refresh the service IP block: %v", err)
	}

	// ensure that ranges are assigned
	for family, snapshot := range snapshotByFamily {
		if snapshot.Range == "" {
			snapshot.Range = c.networkByFamily[family].String()
		}
	}

	// Create an allocator because it is easy to use.
	for family, snapshot := range snapshotByFamily {
		stored, err := ipallocator.NewFromSnapshot(snapshot)
		if err != nil {
			return fmt.Errorf("unable to rebuild allocator from snapshots for family:%v with error:%v", family, err)
		}

		storedByFamily[family] = stored
	}

	rebuiltByFamily := make(map[v1.IPFamily]*ipallocator.Range)

	for family, network := range c.networkByFamily {
		rebuilt, err := ipallocator.NewInMemory(network)
		if err != nil {
			return fmt.Errorf("unable to create CIDR range for family %v: %v", family, err)
		}

		rebuiltByFamily[family] = rebuilt
	}
	// We explicitly send no resource version, since the resource version
	// of 'snapshot' is from a different collection, it's not comparable to
	// the service collection. The caching layer keeps per-collection RVs,
	// and this is proper, since in theory the collections could be hosted
	// in separate etcd (or even non-etcd) instances.
	list, err := c.serviceClient.Services(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{})
	if err != nil {
		return fmt.Errorf("unable to refresh the service IP block: %v", err)
	}

	getFamilyByIP := func(ip net.IP) v1.IPFamily {
		if netutils.IsIPv6(ip) {
			return v1.IPv6Protocol
		}
		return v1.IPv4Protocol
	}

	// Check every Service's ClusterIP, and rebuild the state as we think it should be.
	for _, svc := range list.Items {
		if !helper.IsServiceIPSet(&svc) {
			// didn't need a cluster IP
			continue
		}

		for _, ip := range svc.Spec.ClusterIPs {
			ip := netutils.ParseIPSloppy(ip)
			if ip == nil {
				// cluster IP is corrupt
				c.recorder.Eventf(&svc, nil, v1.EventTypeWarning, "ClusterIPNotValid", "ClusterIPValidation", "Cluster IP %s is not a valid IP; please recreate service", ip)
				runtime.HandleError(fmt.Errorf("the cluster IP %s for service %s/%s is not a valid IP; please recreate", ip, svc.Name, svc.Namespace))
				continue
			}

			family := getFamilyByIP(ip)
			if _, ok := rebuiltByFamily[family]; !ok {
				// this service is using an IPFamily no longer configured on cluster
				c.recorder.Eventf(&svc, nil, v1.EventTypeWarning, "ClusterIPNotValid", "ClusterIPValidation", "Cluster IP %s(%s) is of ip family that is no longer configured on cluster; please recreate service", ip, family)
				runtime.HandleError(fmt.Errorf("the cluster IP %s(%s) for service %s/%s is of ip family that is no longer configured on cluster; please recreate", ip, family, svc.Name, svc.Namespace))
				continue
			}

			// mark it as in-use
			actualAlloc := rebuiltByFamily[family]
			switch err := actualAlloc.Allocate(ip); err {
			case nil:
				actualStored := storedByFamily[family]
				if actualStored.Has(ip) {
					// remove it from the old set, so we can find leaks
					actualStored.Release(ip)
				} else {
					// cluster IP doesn't seem to be allocated
					c.recorder.Eventf(&svc, nil, v1.EventTypeWarning, "ClusterIPNotAllocated", "ClusterIPAllocation", "Cluster IP [%v]:%s is not allocated; repairing", family, ip)
					runtime.HandleError(fmt.Errorf("the cluster IP [%v]:%s for service %s/%s is not allocated; repairing", family, ip, svc.Name, svc.Namespace))
				}
				delete(c.leaksByFamily[family], ip.String()) // it is used, so it can't be leaked
			case ipallocator.ErrAllocated:
				// cluster IP is duplicate
				c.recorder.Eventf(&svc, nil, v1.EventTypeWarning, "ClusterIPAlreadyAllocated", "ClusterIPAllocation", "Cluster IP [%v]:%s was assigned to multiple services; please recreate service", family, ip)
				runtime.HandleError(fmt.Errorf("the cluster IP [%v]:%s for service %s/%s was assigned to multiple services; please recreate", family, ip, svc.Name, svc.Namespace))
			case err.(*ipallocator.ErrNotInRange):
				// cluster IP is out of range
				c.recorder.Eventf(&svc, nil, v1.EventTypeWarning, "ClusterIPOutOfRange", "ClusterIPAllocation", "Cluster IP [%v]:%s is not within the service CIDR %s; please recreate service", family, ip, c.networkByFamily[family])
				runtime.HandleError(fmt.Errorf("the cluster IP [%v]:%s for service %s/%s is not within the service CIDR %s; please recreate", family, ip, svc.Name, svc.Namespace, c.networkByFamily[family]))
			case ipallocator.ErrFull:
				// somehow we are out of IPs
				cidr := actualAlloc.CIDR()
				c.recorder.Eventf(&svc, nil, v1.EventTypeWarning, "ServiceCIDRFull", "ClusterIPAllocation", "Service CIDR %v is full; you must widen the CIDR in order to create new services for Cluster IP [%v]:%s", cidr, family, ip)
				return fmt.Errorf("the service CIDR %v is full; you must widen the CIDR in order to create new services for Cluster IP [%v]:%s", cidr, family, ip)
			default:
				c.recorder.Eventf(&svc, nil, v1.EventTypeWarning, "UnknownError", "ClusterIPAllocation", "Unable to allocate cluster IP [%v]:%s due to an unknown error", family, ip)
				return fmt.Errorf("unable to allocate cluster IP [%v]:%s for service %s/%s due to an unknown error, exiting: %v", family, ip, svc.Name, svc.Namespace, err)
			}
		}
	}

	// leak check
	for family, leaks := range c.leaksByFamily {
		c.checkLeaked(leaks, storedByFamily[family], rebuiltByFamily[family])
	}

	// save logic
	// Blast the rebuilt state into storage.
	for family, rebuilt := range rebuiltByFamily {
		err = c.saveSnapShot(rebuilt, c.allocatorByFamily[family], snapshotByFamily[family])
		if err != nil {
			return err
		}
	}

	return nil
}


saveSnapShot

func (c *Repair) saveSnapShot(rebuilt *ipallocator.Range, alloc rangeallocation.RangeRegistry, snapshot *api.RangeAllocation) error {
	if err := rebuilt.Snapshot(snapshot); err != nil {
		return fmt.Errorf("unable to snapshot the updated service IP allocations: %v", err)
	}
	if err := alloc.CreateOrUpdate(snapshot); err != nil {
		if errors.IsConflict(err) {
			return err
		}
		return fmt.Errorf("unable to persist the updated service IP allocations: %v", err)
	}

	return nil
}


checkLeaked

func (c *Repair) checkLeaked(leaks map[string]int, stored ipallocator.Interface, rebuilt *ipallocator.Range) {
	// Check for IPs that are left in the old set.  They appear to have been leaked.
	stored.ForEach(func(ip net.IP) {
		count, found := leaks[ip.String()]
		switch {
		case !found:
			// flag it to be cleaned up after any races (hopefully) are gone
			runtime.HandleError(fmt.Errorf("the cluster IP %s may have leaked: flagging for later clean up", ip))
			count = numRepairsBeforeLeakCleanup - 1
			fallthrough
		case count > 0:
			// pretend it is still in use until count expires
			leaks[ip.String()] = count - 1
			if err := rebuilt.Allocate(ip); err != nil {
				runtime.HandleError(fmt.Errorf("the cluster IP %s may have leaked, but can not be allocated: %v", ip, err))
			}
		default:
			// do not add it to the rebuilt set, which means it will be available for reuse
			runtime.HandleError(fmt.Errorf("the cluster IP %s appears to have leaked: cleaning up", ip))
		}
	})
}


NewRepairIPAddress

// NewRepair creates a controller that periodically ensures that all clusterIPs are uniquely allocated across the cluster
// and generates informational warnings for a cluster that is not in sync.
func NewRepairIPAddress(interval time.Duration,
	client kubernetes.Interface,
	network *net.IPNet,
	secondaryNetwork *net.IPNet,
	serviceInformer coreinformers.ServiceInformer,
	ipAddressInformer networkinginformers.IPAddressInformer) *RepairIPAddress {
	eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1()})
	recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, "ipallocator-repair-controller")

	networkByFamily := make(map[netutils.IPFamily]*net.IPNet)
	primary := netutils.IPFamilyOfCIDR(network)
	networkByFamily[primary] = network
	if secondaryNetwork != nil {
		secondary := netutils.IPFamilyOfCIDR(secondaryNetwork)
		networkByFamily[secondary] = secondaryNetwork
	}

	r := &RepairIPAddress{
		interval:         interval,
		client:           client,
		networkByFamily:  networkByFamily,
		serviceLister:    serviceInformer.Lister(),
		servicesSynced:   serviceInformer.Informer().HasSynced,
		ipAddressLister:  ipAddressInformer.Lister(),
		ipAddressSynced:  ipAddressInformer.Informer().HasSynced,
		svcQueue:         workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "services"),
		ipQueue:          workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ipaddresses"),
		workerLoopPeriod: time.Second,
		broadcaster:      eventBroadcaster,
		recorder:         recorder,
		clock:            clock.RealClock{},
	}

	serviceInformer.Informer().AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			key, err := cache.MetaNamespaceKeyFunc(obj)
			if err == nil {
				r.svcQueue.Add(key)
			}
		},
		UpdateFunc: func(old interface{}, new interface{}) {
			key, err := cache.MetaNamespaceKeyFunc(new)
			if err == nil {
				r.svcQueue.Add(key)
			}
		},
		DeleteFunc: func(obj interface{}) {
			// IndexerInformer uses a delta queue, therefore for deletes we have to use this
			// key function.
			key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
			if err == nil {
				r.svcQueue.Add(key)
			}
		},
	}, interval)

	ipAddressInformer.Informer().AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			key, err := cache.MetaNamespaceKeyFunc(obj)
			if err == nil {
				r.ipQueue.Add(key)
			}
		},
		UpdateFunc: func(old interface{}, new interface{}) {
			key, err := cache.MetaNamespaceKeyFunc(new)
			if err == nil {
				r.ipQueue.Add(key)
			}
		},
		DeleteFunc: func(obj interface{}) {
			// IndexerInformer uses a delta queue, therefore for deletes we have to use this
			// key function.
			key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
			if err == nil {
				r.ipQueue.Add(key)
			}
		},
	}, interval)

	return r
}


RunUntil

// RunUntil starts the controller until the provided ch is closed.
func (r *RepairIPAddress) RunUntil(onFirstSuccess func(), stopCh chan struct{}) {
	defer r.ipQueue.ShutDown()
	defer r.svcQueue.ShutDown()
	r.broadcaster.StartRecordingToSink(stopCh)
	defer r.broadcaster.Shutdown()

	klog.Info("Starting ipallocator-repair-controller")
	defer klog.Info("Shutting down ipallocator-repair-controller")

	if !cache.WaitForNamedCacheSync("ipallocator-repair-controller", stopCh, r.ipAddressSynced, r.servicesSynced) {
		return
	}

	// First sync goes through all the Services and IPAddresses in the cache,
	// once synced, it signals the main loop and works using the handlers, since
	// it's less expensive and more optimal.
	if err := r.runOnce(); err != nil {
		runtime.HandleError(err)
		return
	}
	onFirstSuccess()

	for i := 0; i < workers; i++ {
		go wait.Until(r.ipWorker, r.workerLoopPeriod, stopCh)
		go wait.Until(r.svcWorker, r.workerLoopPeriod, stopCh)
	}

	<-stopCh
}


runOnce

// runOnce verifies the state of the ClusterIP allocations and returns an error if an unrecoverable problem occurs.
func (r *RepairIPAddress) runOnce() error {
	return retry.RetryOnConflict(retry.DefaultBackoff, r.doRunOnce)
}


doRunOnce

// doRunOnce verifies the state of the ClusterIP allocations and returns an error if an unrecoverable problem occurs.
func (r *RepairIPAddress) doRunOnce() error {
	services, err := r.serviceLister.List(labels.Everything())
	if err != nil {
		return fmt.Errorf("unable to refresh the service IP block: %v", err)
	}

	// Check every Service's ClusterIP, and rebuild the state as we think it should be.
	for _, svc := range services {
		key, err := cache.MetaNamespaceKeyFunc(svc)
		if err != nil {
			return err
		}
		err = r.syncService(key)
		if err != nil {
			return err
		}
	}

	// We have checked that every Service has its corresponding IP.
	// Check that there is no IP created by the allocator without
	// a Service associated.
	ipLabelSelector := labels.Set(map[string]string{
		networkingv1alpha1.LabelManagedBy: ipallocator.ControllerName,
	}).AsSelectorPreValidated()
	ipAddresses, err := r.ipAddressLister.List(ipLabelSelector)
	if err != nil {
		return fmt.Errorf("unable to refresh the IPAddress block: %v", err)
	}
	// Check every IPAddress matches the corresponding Service, and rebuild the state as we think it should be.
	for _, ipAddress := range ipAddresses {
		key, err := cache.MetaNamespaceKeyFunc(ipAddress)
		if err != nil {
			return err
		}
		err = r.syncIPAddress(key)
		if err != nil {
			return err
		}
	}

	return nil
}


svcWorker

func (r *RepairIPAddress) svcWorker() {
	for r.processNextWorkSvc() {
	}
}


processNextWorkSvc

func (r *RepairIPAddress) processNextWorkSvc() bool {
	eKey, quit := r.svcQueue.Get()
	if quit {
		return false
	}
	defer r.svcQueue.Done(eKey)

	err := r.syncService(eKey.(string))
	r.handleSvcErr(err, eKey)

	return true
}


handleSvcErr

func (r *RepairIPAddress) handleSvcErr(err error, key interface{}) {
	if err == nil {
		r.svcQueue.Forget(key)
		return
	}

	if r.svcQueue.NumRequeues(key) < maxRetries {
		klog.V(2).InfoS("Error syncing Service, retrying", "service", key, "err", err)
		r.svcQueue.AddRateLimited(key)
		return
	}

	klog.Warningf("Dropping Service %q out of the queue: %v", key, err)
	r.svcQueue.Forget(key)
	runtime.HandleError(err)
}


syncService

// syncServices reconcile the Service ClusterIPs to verify that each one has the corresponding IPAddress object associated
func (r *RepairIPAddress) syncService(key string) error {
	var syncError error
	namespace, name, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		return err
	}
	svc, err := r.serviceLister.Services(namespace).Get(name)
	if err != nil {
		// nothing to do
		return nil
	}
	if !helper.IsServiceIPSet(svc) {
		// didn't need a ClusterIP
		return nil
	}

	for _, clusterIP := range svc.Spec.ClusterIPs {
		ip := netutils.ParseIPSloppy(clusterIP)
		if ip == nil {
			// ClusterIP is corrupt, ClusterIPs are already validated, but double checking here
			// in case there are some inconsistencies with the parsers
			r.recorder.Eventf(svc, nil, v1.EventTypeWarning, "ClusterIPNotValid", "ClusterIPValidation", "Cluster IP %s is not a valid IP; please recreate Service", ip)
			runtime.HandleError(fmt.Errorf("the ClusterIP %s for Service %s/%s is not a valid IP; please recreate Service", ip, svc.Namespace, svc.Name))
			continue
		}

		family := netutils.IPFamilyOf(ip)
		v1Family := getFamilyByIP(ip)
		network, ok := r.networkByFamily[family]
		if !ok {
			// this service is using an IPFamily no longer configured on cluster
			r.recorder.Eventf(svc, nil, v1.EventTypeWarning, "ClusterIPNotValid", "ClusterIPValidation", "Cluster IP %s(%s) is of ip family that is no longer configured on cluster; please recreate Service", ip, v1Family)
			runtime.HandleError(fmt.Errorf("the ClusterIP [%v]: %s for Service %s/%s is of ip family that is no longer configured on cluster; please recreate Service", v1Family, ip, svc.Namespace, svc.Name))
			continue
		}
		if !network.Contains(ip) {
			// ClusterIP is out of range
			r.recorder.Eventf(svc, nil, v1.EventTypeWarning, "ClusterIPOutOfRange", "ClusterIPAllocation", "Cluster IP [%v]: %s is not within the configured Service CIDR %s; please recreate service", v1Family, ip, network.String())
			runtime.HandleError(fmt.Errorf("the ClusterIP [%v]: %s for Service %s/%s is not within the service CIDR %s; please recreate", v1Family, ip, svc.Namespace, svc.Name, network.String()))
			continue
		}

		// Get the IPAddress object associated to the ClusterIP
		ipAddress, err := r.ipAddressLister.Get(ip.String())
		if apierrors.IsNotFound(err) {
			// ClusterIP doesn't seem to be allocated, create it.
			r.recorder.Eventf(svc, nil, v1.EventTypeWarning, "ClusterIPNotAllocated", "ClusterIPAllocation", "Cluster IP [%v]: %s is not allocated; repairing", v1Family, ip)
			runtime.HandleError(fmt.Errorf("the ClusterIP [%v]: %s for Service %s/%s is not allocated; repairing", v1Family, ip, svc.Namespace, svc.Name))
			_, err := r.client.NetworkingV1alpha1().IPAddresses().Create(context.Background(), newIPAddress(ip.String(), svc), metav1.CreateOptions{})
			if err != nil {
				return err
			}
			continue
		}
		if err != nil {
			r.recorder.Eventf(svc, nil, v1.EventTypeWarning, "UnknownError", "ClusterIPAllocation", "Unable to allocate ClusterIP [%v]: %s due to an unknown error", v1Family, ip)
			return fmt.Errorf("unable to allocate ClusterIP [%v]: %s for Service %s/%s due to an unknown error, will retry later: %v", v1Family, ip, svc.Namespace, svc.Name, err)
		}

		// IPAddress that belongs to a Service must reference a Service
		if ipAddress.Spec.ParentRef.Group != "" ||
			ipAddress.Spec.ParentRef.Resource != "services" {
			r.recorder.Eventf(svc, nil, v1.EventTypeWarning, "ClusterIPNotAllocated", "ClusterIPAllocation", "the ClusterIP [%v]: %s for Service %s/%s has a wrong reference; repairing", v1Family, ip, svc.Namespace, svc.Name)
			if err := r.recreateIPAddress(ipAddress.Name, svc); err != nil {
				return err
			}
			continue
		}

		// IPAddress that belongs to a Service must reference the current Service
		if ipAddress.Spec.ParentRef.Namespace != svc.Namespace ||
			ipAddress.Spec.ParentRef.Name != svc.Name {
			// verify that there are no two Services with the same IP, otherwise
			// it will keep deleting and recreating the same IPAddress changing the reference
			refService, err := r.serviceLister.Services(ipAddress.Spec.ParentRef.Namespace).Get(ipAddress.Spec.ParentRef.Name)
			if err != nil {
				r.recorder.Eventf(svc, nil, v1.EventTypeWarning, "ClusterIPNotAllocated", "ClusterIPAllocation", "the ClusterIP [%v]: %s for Service %s/%s has a wrong reference; repairing", v1Family, ip, svc.Namespace, svc.Name)
				if err := r.recreateIPAddress(ipAddress.Name, svc); err != nil {
					return err
				}
				continue
			}
			// the IPAddress is duplicate but current Service is not the referenced, it has to be recreated
			for _, clusterIP := range refService.Spec.ClusterIPs {
				if ipAddress.Name == clusterIP {
					r.recorder.Eventf(svc, nil, v1.EventTypeWarning, "ClusterIPAlreadyAllocated", "ClusterIPAllocation", "Cluster IP [%v]:%s was assigned to multiple services; please recreate service", family, ip)
					runtime.HandleError(fmt.Errorf("the cluster IP [%v]:%s for service %s/%s was assigned to other services %s/%s; please recreate", family, ip, svc.Namespace, svc.Name, refService.Namespace, refService.Name))
					break
				}
			}
		}

		// IPAddress must have the corresponding labels assigned by the allocator
		if !verifyIPAddressLabels(ipAddress) {
			if err := r.recreateIPAddress(ipAddress.Name, svc); err != nil {
				return err
			}
			continue
		}

	}
	return syncError
}


recreateIPAddress

func (r *RepairIPAddress) recreateIPAddress(name string, svc *v1.Service) error {
	err := r.client.NetworkingV1alpha1().IPAddresses().Delete(context.Background(), name, metav1.DeleteOptions{})
	if err != nil && !apierrors.IsNotFound(err) {
		return err
	}
	_, err = r.client.NetworkingV1alpha1().IPAddresses().Create(context.Background(), newIPAddress(name, svc), metav1.CreateOptions{})
	if err != nil {
		return err
	}
	return nil
}


ipWorker

func (r *RepairIPAddress) ipWorker() {
	for r.processNextWorkIp() {
	}
}


processNextWorkIp

func (r *RepairIPAddress) processNextWorkIp() bool {
	eKey, quit := r.ipQueue.Get()
	if quit {
		return false
	}
	defer r.ipQueue.Done(eKey)

	err := r.syncIPAddress(eKey.(string))
	r.handleIpErr(err, eKey)

	return true
}


handleIpErr

func (r *RepairIPAddress) handleIpErr(err error, key interface{}) {
	if err == nil {
		r.ipQueue.Forget(key)
		return
	}

	if r.ipQueue.NumRequeues(key) < maxRetries {
		klog.V(2).InfoS("Error syncing Service, retrying", "service", key, "err", err)
		r.ipQueue.AddRateLimited(key)
		return
	}

	klog.Warningf("Dropping Service %q out of the queue: %v", key, err)
	r.ipQueue.Forget(key)
	runtime.HandleError(err)
}


syncIPAddress

// syncIPAddress verify that the IPAddress that are owned by the ipallocator controller reference an existing Service
// to avoid leaking IPAddresses. IPAddresses that are owned by other controllers are not processed to avoid hotloops.
// IPAddress that reference Services and are part of the ClusterIP are validated in the syncService loop.
func (r *RepairIPAddress) syncIPAddress(key string) error {
	ipAddress, err := r.ipAddressLister.Get(key)
	if err != nil {
		// nothing to do
		return nil
	}

	// not mananged by this controller
	if !managedByController(ipAddress) {
		return nil
	}

	// does not reference a Service but created by the service allocator, something else have changed it, delete it
	if ipAddress.Spec.ParentRef.Group != "" ||
		ipAddress.Spec.ParentRef.Resource != "services" {
		runtime.HandleError(fmt.Errorf("IPAddress %s appears to have been modified, not referencing a Service %v: cleaning up", ipAddress.Name, ipAddress.Spec.ParentRef))
		r.recorder.Eventf(ipAddress, nil, v1.EventTypeWarning, "IPAddressNotAllocated", "IPAddressAllocation", "IPAddress %s appears to have been modified, not referencing a Service %v: cleaning up", ipAddress.Name, ipAddress.Spec.ParentRef)
		err := r.client.NetworkingV1alpha1().IPAddresses().Delete(context.Background(), ipAddress.Name, metav1.DeleteOptions{})
		if err != nil && !apierrors.IsNotFound(err) {
			return err
		}
		return nil
	}

	svc, err := r.serviceLister.Services(ipAddress.Spec.ParentRef.Namespace).Get(ipAddress.Spec.ParentRef.Name)
	if apierrors.IsNotFound(err) {
		// cleaning all IPAddress without an owner reference IF the time since it was created is greater than 60 seconds (default timeout value on the kube-apiserver)
		// This is required because during the Service creation there is a time that the IPAddress object exists but the Service is still being created
		// Assume that CreationTimestamp exists.
		ipLifetime := r.clock.Now().Sub(ipAddress.CreationTimestamp.Time)
		gracePeriod := 60 * time.Second
		if ipLifetime > gracePeriod {
			runtime.HandleError(fmt.Errorf("IPAddress %s appears to have leaked: cleaning up", ipAddress.Name))
			r.recorder.Eventf(ipAddress, nil, v1.EventTypeWarning, "IPAddressNotAllocated", "IPAddressAllocation", "IPAddress: %s for Service %s/%s appears to have leaked: cleaning up", ipAddress.Name, ipAddress.Spec.ParentRef.Namespace, ipAddress.Spec.ParentRef.Name)
			err := r.client.NetworkingV1alpha1().IPAddresses().Delete(context.Background(), ipAddress.Name, metav1.DeleteOptions{})
			if err != nil && !apierrors.IsNotFound(err) {
				return err
			}
		}
		// requeue after the grace period
		r.ipQueue.AddAfter(key, gracePeriod-ipLifetime)
		return nil
	}
	if err != nil {
		runtime.HandleError(fmt.Errorf("unable to get parent Service for IPAddress %s due to an unknown error: %v", ipAddress, err))
		r.recorder.Eventf(ipAddress, nil, v1.EventTypeWarning, "UnknownError", "IPAddressAllocation", "Unable to get parent Service for IPAddress %s due to an unknown error", ipAddress)
		return err
	}
	// The service exists, we have checked in previous loop that all Service to IPAddress are correct
	// but we also have to check the reverse, that the IPAddress to Service relation is correct
	for _, clusterIP := range svc.Spec.ClusterIPs {
		if ipAddress.Name == clusterIP {
			return nil
		}
	}
	runtime.HandleError(fmt.Errorf("the IPAddress: %s for Service %s/%s has a wrong reference %#v; cleaning up", ipAddress.Name, svc.Name, svc.Namespace, ipAddress.Spec.ParentRef))
	r.recorder.Eventf(ipAddress, nil, v1.EventTypeWarning, "IPAddressWrongReference", "IPAddressAllocation", "IPAddress: %s for Service %s/%s has a wrong reference; cleaning up", ipAddress.Name, svc.Namespace, svc.Name)
	err = r.client.NetworkingV1alpha1().IPAddresses().Delete(context.Background(), ipAddress.Name, metav1.DeleteOptions{})
	if err != nil && !apierrors.IsNotFound(err) {
		return err
	}
	return nil

}


newIPAddress

func newIPAddress(name string, svc *v1.Service) *networkingv1alpha1.IPAddress {
	family := string(v1.IPv4Protocol)
	if netutils.IsIPv6String(name) {
		family = string(v1.IPv6Protocol)
	}
	return &networkingv1alpha1.IPAddress{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
			Labels: map[string]string{
				networkingv1alpha1.LabelIPAddressFamily: family,
				networkingv1alpha1.LabelManagedBy:       ipallocator.ControllerName,
			},
		},
		Spec: networkingv1alpha1.IPAddressSpec{
			ParentRef: serviceToRef(svc),
		},
	}
}


serviceToRef

func serviceToRef(svc *v1.Service) *networkingv1alpha1.ParentReference {
	if svc == nil {
		return nil
	}

	return &networkingv1alpha1.ParentReference{
		Group:     "",
		Resource:  "services",
		Namespace: svc.Namespace,
		Name:      svc.Name,
		UID:       svc.UID,
	}
}


getFamilyByIP

func getFamilyByIP(ip net.IP) v1.IPFamily {
	if netutils.IsIPv6(ip) {
		return v1.IPv6Protocol
	}
	return v1.IPv4Protocol
}


managedByController

// managedByController returns true if the controller of the provided
// EndpointSlices is the EndpointSlice controller.
func managedByController(ip *networkingv1alpha1.IPAddress) bool {
	managedBy, ok := ip.Labels[networkingv1alpha1.LabelManagedBy]
	if !ok {
		return false
	}
	return managedBy == ipallocator.ControllerName
}


verifyIPAddressLabels

func verifyIPAddressLabels(ip *networkingv1alpha1.IPAddress) bool {
	labelFamily, ok := ip.Labels[networkingv1alpha1.LabelIPAddressFamily]
	if !ok {
		return false
	}

	family := string(v1.IPv4Protocol)
	if netutils.IsIPv6String(ip.Name) {
		family = string(v1.IPv6Protocol)
	}
	if family != labelFamily {
		return false
	}
	return managedByController(ip)
}


Error

func (e *ErrNotInRange) Error() string {
	return fmt.Sprintf("the provided IP (%v) is not in the valid range. The range of valid IPs is %s", e.IP, e.ValidRange)
}


NewIPAllocator

// NewIPAllocator returns an IP allocator associated to a network range
// that use the IPAddress objectto track the assigned IP addresses,
// using an informer cache as storage.
func NewIPAllocator(
	cidr *net.IPNet,
	client networkingv1alpha1client.NetworkingV1alpha1Interface,
	ipAddressInformer networkingv1alpha1informers.IPAddressInformer,
) (*Allocator, error) {
	prefix, err := netip.ParsePrefix(cidr.String())
	if err != nil {
		return nil, err
	}

	if prefix.Addr().Is6() && prefix.Bits() < 64 {
		return nil, fmt.Errorf("shortest allowed prefix length for service CIDR is 64, got %d", prefix.Bits())
	}

	// TODO: use the utils/net function once is available
	size := hostsPerNetwork(cidr)
	var family api.IPFamily
	if netutils.IsIPv6CIDR(cidr) {
		family = api.IPv6Protocol
	} else {
		family = api.IPv4Protocol
	}
	// Caching the first, offset and last addresses allows to optimize
	// the search loops by using the netip.Addr iterator instead
	// of having to do conversions with IP addresses.
	// Don't allocate the network's ".0" address.
	ipFirst := prefix.Masked().Addr().Next()
	if err != nil {
		return nil, err
	}
	// Use the broadcast address as last address for IPv6
	ipLast, err := broadcastAddress(prefix)
	if err != nil {
		return nil, err
	}
	// For IPv4 don't use the network's broadcast address
	if family == api.IPv4Protocol {
		ipLast = ipLast.Prev()
	}
	// KEP-3070: Reserve Service IP Ranges For Dynamic and Static IP Allocation
	// calculate the subrange offset
	rangeOffset := calculateRangeOffset(cidr)
	offsetAddress, err := addOffsetAddress(ipFirst, uint64(rangeOffset))
	if err != nil {
		return nil, err
	}
	a := &Allocator{
		cidr:            cidr,
		prefix:          prefix,
		firstAddress:    ipFirst,
		lastAddress:     ipLast,
		rangeOffset:     rangeOffset,
		offsetAddress:   offsetAddress,
		size:            size,
		family:          family,
		client:          client,
		ipAddressLister: ipAddressInformer.Lister(),
		ipAddressSynced: ipAddressInformer.Informer().HasSynced,
		metrics:         &emptyMetricsRecorder{}, // disabled by default
		metricLabel:     cidr.String(),
		rand:            rand.New(rand.NewSource(time.Now().UnixNano())),
	}

	return a, nil
}


createIPAddress

func (a *Allocator) createIPAddress(name string, svc *api.Service, scope string) error {
	ipAddress := networkingv1alpha1.IPAddress{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
			Labels: map[string]string{
				networkingv1alpha1.LabelIPAddressFamily: string(a.IPFamily()),
				networkingv1alpha1.LabelManagedBy:       ControllerName,
			},
		},
		Spec: networkingv1alpha1.IPAddressSpec{
			ParentRef: serviceToRef(svc),
		},
	}
	_, err := a.client.IPAddresses().Create(context.Background(), &ipAddress, metav1.CreateOptions{})
	if err != nil {
		// update metrics
		a.metrics.incrementAllocationErrors(a.metricLabel, scope)
		if apierrors.IsAlreadyExists(err) {
			return ErrAllocated
		}
		return err
	}
	// update metrics
	a.metrics.incrementAllocations(a.metricLabel, scope)
	a.metrics.setAllocated(a.metricLabel, a.Used())
	a.metrics.setAvailable(a.metricLabel, a.Free())
	return nil
}


Allocate

// Allocate attempts to reserve the provided IP. ErrNotInRange or
// ErrAllocated will be returned if the IP is not valid for this range
// or has already been reserved.  ErrFull will be returned if there
// are no addresses left.
// Only for testing, it will fail to create the IPAddress object because
// the Service reference is required.
func (a *Allocator) Allocate(ip net.IP) error {
	return a.AllocateService(nil, ip)
}


AllocateService

// AllocateService attempts to reserve the provided IP. ErrNotInRange or
// ErrAllocated will be returned if the IP is not valid for this range
// or has already been reserved.  ErrFull will be returned if there
// are no addresses left.
func (a *Allocator) AllocateService(svc *api.Service, ip net.IP) error {
	return a.allocateService(svc, ip, dryRunFalse)
}


allocateService

func (a *Allocator) allocateService(svc *api.Service, ip net.IP, dryRun bool) error {
	if !a.ipAddressSynced() {
		return fmt.Errorf("allocator not ready")
	}
	addr, err := netip.ParseAddr(ip.String())
	if err != nil {
		return err
	}

	// check address is within the range of available addresses
	if addr.Less(a.firstAddress) || // requested address is lower than the first address in the subnet
		a.lastAddress.Less(addr) { // the last address in the subnet is lower than the requested address
		if !dryRun {
			// update metrics
			a.metrics.incrementAllocationErrors(a.metricLabel, "static")
		}
		return &ErrNotInRange{ip, a.prefix.String()}
	}
	if dryRun {
		return nil
	}
	return a.createIPAddress(ip.String(), svc, "static")
}


AllocateNext

// AllocateNext return an IP address that wasn't allocated yet.
// Only for testing, it will fail to create the IPAddress object because
// the Service reference is required.
func (a *Allocator) AllocateNext() (net.IP, error) {
	return a.AllocateNextService(nil)
}


AllocateNextService

// AllocateNext return an IP address that wasn't allocated yet.
func (a *Allocator) AllocateNextService(svc *api.Service) (net.IP, error) {
	return a.allocateNextService(svc, dryRunFalse)
}


allocateNextService

// allocateNextService tries to allocate a free IP address within the subnet.
// If the subnet is big enough, it partitions the subnet into two subranges,
// delimited by a.rangeOffset.
// It tries to allocate a free IP address from the upper subnet first and
// falls back to the lower subnet.
// It starts allocating from a random IP within each range.
func (a *Allocator) allocateNextService(svc *api.Service, dryRun bool) (net.IP, error) {
	if !a.ipAddressSynced() {
		return nil, fmt.Errorf("allocator not ready")
	}
	if dryRun {
		// Don't bother finding a free value. It's racy and not worth the
		// effort to plumb any further.
		return a.CIDR().IP, nil
	}

	trace := utiltrace.New("allocate dynamic ClusterIP address")
	defer trace.LogIfLong(500 * time.Millisecond)

	// rand.Int63n panics for n <= 0 so we need to avoid problems when
	// converting from uint64 to int64
	rangeSize := a.size - uint64(a.rangeOffset)
	var offset uint64
	switch {
	case rangeSize >= math.MaxInt64:
		offset = rand.Uint64()
	case rangeSize == 0:
		return net.IP{}, ErrFull
	default:
		offset = uint64(a.rand.Int63n(int64(rangeSize)))
	}
	iterator := ipIterator(a.offsetAddress, a.lastAddress, offset)
	ip, err := a.allocateFromRange(iterator, svc)
	if err == nil {
		return ip, nil
	}
	// check the lower range
	if a.rangeOffset != 0 {
		offset = uint64(a.rand.Intn(a.rangeOffset))
		iterator = ipIterator(a.firstAddress, a.offsetAddress.Prev(), offset)
		ip, err = a.allocateFromRange(iterator, svc)
		if err == nil {
			return ip, nil
		}
	}
	// update metrics
	a.metrics.incrementAllocationErrors(a.metricLabel, "dynamic")
	return net.IP{}, ErrFull
}


ipIterator

// IP iterator allows to iterate over all the IP addresses
// in a range defined by the start and last address.
// It starts iterating at the address position defined by the offset.
// It returns an invalid address to indicate it hasfinished.
func ipIterator(first netip.Addr, last netip.Addr, offset uint64) func() netip.Addr {
	// There are no modulo operations for IP addresses
	modulo := func(addr netip.Addr) netip.Addr {
		if addr.Compare(last) == 1 {
			return first
		}
		return addr
	}
	next := func(addr netip.Addr) netip.Addr {
		return modulo(addr.Next())
	}
	start, err := addOffsetAddress(first, offset)
	if err != nil {
		return func() netip.Addr { return netip.Addr{} }
	}
	start = modulo(start)
	ip := start
	seen := false
	return func() netip.Addr {
		value := ip
		// is the last or the first iteration
		if value == start {
			if seen {
				return netip.Addr{}
			}
			seen = true
		}
		ip = next(ip)
		return value
	}

}


allocateFromRange

// allocateFromRange allocates an empty IP address from the range of
// IPs between the first and last address (both included), starting
// from the start address.
// TODO: this is a linear search, it can be optimized.
func (a *Allocator) allocateFromRange(iterator func() netip.Addr, svc *api.Service) (net.IP, error) {
	for {
		ip := iterator()
		if !ip.IsValid() {
			break
		}
		name := ip.String()
		_, err := a.ipAddressLister.Get(name)
		// continue if ip already exist
		if err == nil {
			continue
		}
		if !apierrors.IsNotFound(err) {
			klog.Infof("unexpected error: %v", err)
			continue
		}
		// address is not present on the cache, try to allocate it
		err = a.createIPAddress(name, svc, "dynamic")
		// an error can happen if there is a race and our informer was not updated
		// swallow the error and try with the next IP address
		if err != nil {
			klog.Infof("can not create IPAddress %s: %v", name, err)
			continue
		}
		return ip.AsSlice(), nil
	}
	return net.IP{}, ErrFull
}


Release

// Release releases the IP back to the pool. Releasing an
// unallocated IP or an IP out of the range is a no-op and
// returns no error.
func (a *Allocator) Release(ip net.IP) error {
	return a.release(ip, dryRunFalse)
}


release

func (a *Allocator) release(ip net.IP, dryRun bool) error {
	if !a.ipAddressSynced() {
		return fmt.Errorf("allocator not ready")
	}
	if dryRun {
		return nil
	}
	name := ip.String()
	// Try to Delete the IPAddress independently of the cache state.
	// The error is ignored for compatibility reasons.
	err := a.client.IPAddresses().Delete(context.Background(), name, metav1.DeleteOptions{})
	if err == nil {
		// update metrics
		a.metrics.setAllocated(a.metricLabel, a.Used())
		a.metrics.setAvailable(a.metricLabel, a.Free())
		return nil
	}
	klog.Infof("error releasing ip %s : %v", name, err)
	return nil
}


ForEach

// ForEach executes the function on each allocated IP
// This is required to satisfy the Allocator Interface only
func (a *Allocator) ForEach(f func(net.IP)) {
	ipLabelSelector := labels.Set(map[string]string{
		networkingv1alpha1.LabelIPAddressFamily: string(a.IPFamily()),
		networkingv1alpha1.LabelManagedBy:       ControllerName,
	}).AsSelectorPreValidated()
	ips, err := a.ipAddressLister.List(ipLabelSelector)
	if err != nil {
		return
	}
	for _, ip := range ips {
		f(netutils.ParseIPSloppy(ip.Name))
	}
}


CIDR

func (a *Allocator) CIDR() net.IPNet {
	return *a.cidr
}


Has

// for testing
func (a *Allocator) Has(ip net.IP) bool {
	// convert IP to name
	name := ip.String()
	ipAddress, err := a.client.IPAddresses().Get(context.Background(), name, metav1.GetOptions{})
	if err != nil || len(ipAddress.Name) == 0 {
		return false
	}
	return true
}


IPFamily

func (a *Allocator) IPFamily() api.IPFamily {
	return a.family
}


Used

// for testing
func (a *Allocator) Used() int {
	ipLabelSelector := labels.Set(map[string]string{
		networkingv1alpha1.LabelIPAddressFamily: string(a.IPFamily()),
		networkingv1alpha1.LabelManagedBy:       ControllerName,
	}).AsSelectorPreValidated()
	ips, err := a.ipAddressLister.List(ipLabelSelector)
	if err != nil {
		return 0
	}
	return len(ips)
}


Free

// for testing
func (a *Allocator) Free() int {
	return int(a.size) - a.Used()
}


Destroy

// Destroy
func (a *Allocator) Destroy() {
}


DryRun

// DryRun
func (a *Allocator) DryRun() Interface {
	return dryRunAllocator{a}
}


EnableMetrics

// EnableMetrics
func (a *Allocator) EnableMetrics() {
	registerMetrics()
	a.metrics = &metricsRecorder{}
}


Allocate

func (dry dryRunAllocator) Allocate(ip net.IP) error {
	return dry.real.allocateService(nil, ip, dryRunTrue)

}


AllocateNext

func (dry dryRunAllocator) AllocateNext() (net.IP, error) {
	return dry.real.allocateNextService(nil, dryRunTrue)
}


Release

func (dry dryRunAllocator) Release(ip net.IP) error {
	return dry.real.release(ip, dryRunTrue)
}


ForEach

func (dry dryRunAllocator) ForEach(cb func(net.IP)) {
	dry.real.ForEach(cb)
}


CIDR

func (dry dryRunAllocator) CIDR() net.IPNet {
	return dry.real.CIDR()
}


IPFamily

func (dry dryRunAllocator) IPFamily() api.IPFamily {
	return dry.real.IPFamily()
}


DryRun

func (dry dryRunAllocator) DryRun() Interface {
	return dry
}


Has

func (dry dryRunAllocator) Has(ip net.IP) bool {
	return dry.real.Has(ip)
}


Destroy

func (dry dryRunAllocator) Destroy() {
}


EnableMetrics

func (dry dryRunAllocator) EnableMetrics() {
}


addOffsetAddress

// addOffsetAddress returns the address at the provided offset within the subnet
// TODO: move it to k8s.io/utils/net, this is the same as current AddIPOffset()
// but using netip.Addr instead of net.IP
func addOffsetAddress(address netip.Addr, offset uint64) (netip.Addr, error) {
	addressBig := big.NewInt(0).SetBytes(address.AsSlice())
	r := big.NewInt(0).Add(addressBig, big.NewInt(int64(offset)))
	addr, ok := netip.AddrFromSlice(r.Bytes())
	if !ok {
		return netip.Addr{}, fmt.Errorf("invalid address %v", r.Bytes())
	}
	return addr, nil
}


hostsPerNetwork

// hostsPerNetwork returns the number of available hosts in a subnet.
// The max number is limited by the size of an uint64.
// Number of hosts is calculated with the formula:
// IPv4: 2^x – 2, not consider network and broadcast address
// IPv6: 2^x - 1, not consider network address
// where x is the number of host bits in the subnet.
func hostsPerNetwork(subnet *net.IPNet) uint64 {
	ones, bits := subnet.Mask.Size()
	// this checks that we are not overflowing an int64
	if bits-ones >= 64 {
		return math.MaxUint64
	}
	max := uint64(1) << uint(bits-ones)
	// Don't use the network's ".0" address,
	if max == 0 {
		return 0
	}
	max--
	if netutils.IsIPv4CIDR(subnet) {
		// Don't use the IPv4 network's broadcast address
		if max == 0 {
			return 0
		}
		max--
	}
	return max
}


broadcastAddress

// broadcastAddress returns the broadcast address of the subnet
// The broadcast address is obtained by setting all the host bits
// in a subnet to 1.
// network 192.168.0.0/24 : subnet bits 24 host bits 32 - 24 = 8
// broadcast address 192.168.0.255
func broadcastAddress(subnet netip.Prefix) (netip.Addr, error) {
	base := subnet.Masked().Addr()
	bytes := base.AsSlice()
	// get all the host bits from the subnet
	n := 8*len(bytes) - subnet.Bits()
	// set all the host bits to 1
	for i := len(bytes) - 1; i >= 0 && n > 0; i-- {
		if n >= 8 {
			bytes[i] = 0xff
			n -= 8
		} else {
			mask := ^uint8(0) >> (8 - n)
			bytes[i] |= mask
			break
		}
	}

	addr, ok := netip.AddrFromSlice(bytes)
	if !ok {
		return netip.Addr{}, fmt.Errorf("invalid address %v", bytes)
	}
	return addr, nil
}


serviceToRef

// serviceToRef obtain the Service Parent Reference
func serviceToRef(svc *api.Service) *networkingv1alpha1.ParentReference {
	if svc == nil {
		return nil
	}

	return &networkingv1alpha1.ParentReference{
		Group:     "",
		Resource:  "services",
		Namespace: svc.Namespace,
		Name:      svc.Name,
		UID:       svc.UID,
	}
}


registerMetrics

func registerMetrics() {
	registerMetricsOnce.Do(func() {
		legacyregistry.MustRegister(clusterIPAllocated)
		legacyregistry.MustRegister(clusterIPAvailable)
		legacyregistry.MustRegister(clusterIPAllocations)
		legacyregistry.MustRegister(clusterIPAllocationErrors)
	})
}


setAllocated

func (m *metricsRecorder) setAllocated(cidr string, allocated int) {
	clusterIPAllocated.WithLabelValues(cidr).Set(float64(allocated))
}


setAvailable

func (m *metricsRecorder) setAvailable(cidr string, available int) {
	clusterIPAvailable.WithLabelValues(cidr).Set(float64(available))
}


incrementAllocations

func (m *metricsRecorder) incrementAllocations(cidr, scope string) {
	clusterIPAllocations.WithLabelValues(cidr, scope).Inc()
}


incrementAllocationErrors

func (m *metricsRecorder) incrementAllocationErrors(cidr, scope string) {
	clusterIPAllocationErrors.WithLabelValues(cidr, scope).Inc()
}


setAllocated

func (*emptyMetricsRecorder) setAllocated(cidr string, allocated int)      {}


setAvailable

func (*emptyMetricsRecorder) setAvailable(cidr string, available int)      {}


incrementAllocations

func (*emptyMetricsRecorder) incrementAllocations(cidr, scope string)      {}


incrementAllocationErrors

func (*emptyMetricsRecorder) incrementAllocationErrors(cidr, scope string) {}


Error

func (e *ErrNotInRange) Error() string {
	return fmt.Sprintf("provided port is not in the valid range. The range of valid ports is %s", e.ValidPorts)
}


New

// New creates a PortAllocator over a net.PortRange, calling allocatorFactory to construct the backing store.
func New(pr net.PortRange, allocatorFactory allocator.AllocatorWithOffsetFactory) (*PortAllocator, error) {
	max := pr.Size
	rangeSpec := pr.String()

	a := &PortAllocator{
		portRange: pr,
		metrics:   &emptyMetricsRecorder{},
	}

	var offset = 0
	if utilfeature.DefaultFeatureGate.Enabled(features.ServiceNodePortStaticSubrange) {
		offset = calculateRangeOffset(pr)
	}

	var err error
	a.alloc, err = allocatorFactory(max, rangeSpec, offset)
	if err != nil {
		return nil, err
	}

	return a, err
}


NewInMemory

// NewInMemory creates an in-memory allocator.
func NewInMemory(pr net.PortRange) (*PortAllocator, error) {
	return New(pr, func(max int, rangeSpec string, offset int) (allocator.Interface, error) {
		return allocator.NewAllocationMapWithOffset(max, rangeSpec, offset), nil
	})
}


NewFromSnapshot

// NewFromSnapshot allocates a PortAllocator and initializes it from a snapshot.
func NewFromSnapshot(snap *api.RangeAllocation) (*PortAllocator, error) {
	pr, err := net.ParsePortRange(snap.Range)
	if err != nil {
		return nil, err
	}
	r, err := NewInMemory(*pr)
	if err != nil {
		return nil, err
	}
	if err := r.Restore(*pr, snap.Data); err != nil {
		return nil, err
	}
	return r, nil
}


Free

// Free returns the count of port left in the range.
func (r *PortAllocator) Free() int {
	return r.alloc.Free()
}


Used

// Used returns the count of ports used in the range.
func (r *PortAllocator) Used() int {
	return r.portRange.Size - r.alloc.Free()
}


Allocate

// Allocate attempts to reserve the provided port. ErrNotInRange or
// ErrAllocated will be returned if the port is not valid for this range
// or has already been reserved.  ErrFull will be returned if there
// are no ports left.
func (r *PortAllocator) Allocate(port int) error {
	ok, offset := r.contains(port)
	if !ok {
		// update metrics
		r.metrics.incrementAllocationErrors("static")

		// include valid port range in error
		validPorts := r.portRange.String()
		return &ErrNotInRange{validPorts}
	}

	allocated, err := r.alloc.Allocate(offset)
	if err != nil {
		// update metrics
		r.metrics.incrementAllocationErrors("static")
		return err
	}
	if !allocated {
		// update metrics
		r.metrics.incrementAllocationErrors("static")
		return ErrAllocated
	}

	// update metrics
	r.metrics.incrementAllocations("static")
	r.metrics.setAllocated(r.Used())
	r.metrics.setAvailable(r.Free())

	return nil
}


AllocateNext

// AllocateNext reserves one of the ports from the pool. ErrFull may
// be returned if there are no ports left.
func (r *PortAllocator) AllocateNext() (int, error) {
	offset, ok, err := r.alloc.AllocateNext()
	if err != nil {
		r.metrics.incrementAllocationErrors("dynamic")
		return 0, err
	}
	if !ok {
		r.metrics.incrementAllocationErrors("dynamic")
		return 0, ErrFull
	}

	// update metrics
	r.metrics.incrementAllocations("dynamic")
	r.metrics.setAllocated(r.Used())
	r.metrics.setAvailable(r.Free())

	return r.portRange.Base + offset, nil
}


ForEach

// ForEach calls the provided function for each allocated port.
func (r *PortAllocator) ForEach(fn func(int)) {
	r.alloc.ForEach(func(offset int) {
		fn(r.portRange.Base + offset)
	})
}


Release

// Release releases the port back to the pool. Releasing an
// unallocated port or a port out of the range is a no-op and
// returns no error.
func (r *PortAllocator) Release(port int) error {
	ok, offset := r.contains(port)
	if !ok {
		klog.Warningf("port is not in the range when release it. port: %v", port)
		return nil
	}

	err := r.alloc.Release(offset)
	if err == nil {
		// update metrics
		r.metrics.setAllocated(r.Used())
		r.metrics.setAvailable(r.Free())
	}
	return err
}


Has

// Has returns true if the provided port is already allocated and a call
// to Allocate(port) would fail with ErrAllocated.
func (r *PortAllocator) Has(port int) bool {
	ok, offset := r.contains(port)
	if !ok {
		return false
	}

	return r.alloc.Has(offset)
}


Snapshot

// Snapshot saves the current state of the pool.
func (r *PortAllocator) Snapshot(dst *api.RangeAllocation) error {
	snapshottable, ok := r.alloc.(allocator.Snapshottable)
	if !ok {
		return fmt.Errorf("not a snapshottable allocator")
	}
	rangeString, data := snapshottable.Snapshot()
	dst.Range = rangeString
	dst.Data = data
	return nil
}


Restore

// Restore restores the pool to the previously captured state. ErrMismatchedNetwork
// is returned if the provided port range doesn't exactly match the previous range.
func (r *PortAllocator) Restore(pr net.PortRange, data []byte) error {
	if pr.String() != r.portRange.String() {
		return ErrMismatchedNetwork
	}
	snapshottable, ok := r.alloc.(allocator.Snapshottable)
	if !ok {
		return fmt.Errorf("not a snapshottable allocator")
	}
	return snapshottable.Restore(pr.String(), data)
}


contains

// contains returns true and the offset if the port is in the range, and false
// and nil otherwise.
func (r *PortAllocator) contains(port int) (bool, int) {
	if !r.portRange.Contains(port) {
		return false, 0
	}

	offset := port - r.portRange.Base
	return true, offset
}


Destroy

// Destroy shuts down internal allocator.
func (r *PortAllocator) Destroy() {
	r.alloc.Destroy()
}


EnableMetrics

// EnableMetrics enables metrics recording.
func (r *PortAllocator) EnableMetrics() {
	registerMetrics()
	r.metrics = &metricsRecorder{}
}


calculateRangeOffset

// calculateRangeOffset estimates the offset used on the range for statically allocation based on
// the following formula `min(max($min, rangeSize/$step), $max)`, described as ~never less than
// $min or more than $max, with a graduated step function between them~. The function returns 0
// if any of the parameters is invalid.
func calculateRangeOffset(pr net.PortRange) int {
	// default values for min(max($min, rangeSize/$step), $max)
	const (
		min  = 16
		max  = 128
		step = 32
	)

	rangeSize := pr.Size
	// offset should always be smaller than the range size
	if rangeSize <= min {
		return 0
	}

	offset := rangeSize / step
	if offset < min {
		return min
	}
	if offset > max {
		return max
	}
	return int(offset)
}


NewRepair

// NewRepair creates a controller that periodically ensures that all ports are uniquely allocated across the cluster
// and generates informational warnings for a cluster that is not in sync.
func NewRepair(interval time.Duration, serviceClient corev1client.ServicesGetter, eventClient eventsv1client.EventsV1Interface, portRange net.PortRange, alloc rangeallocation.RangeRegistry) *Repair {
	eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: eventClient})
	recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, "portallocator-repair-controller")

	return &Repair{
		interval:      interval,
		serviceClient: serviceClient,
		portRange:     portRange,
		alloc:         alloc,
		leaks:         map[int]int{},
		broadcaster:   eventBroadcaster,
		recorder:      recorder,
	}
}


RunUntil

// RunUntil starts the controller until the provided ch is closed.
func (c *Repair) RunUntil(onFirstSuccess func(), stopCh chan struct{}) {
	c.broadcaster.StartRecordingToSink(stopCh)
	defer c.broadcaster.Shutdown()

	var once sync.Once
	wait.Until(func() {
		if err := c.runOnce(); err != nil {
			runtime.HandleError(err)
			return
		}
		once.Do(onFirstSuccess)
	}, c.interval, stopCh)
}


runOnce

// runOnce verifies the state of the port allocations and returns an error if an unrecoverable problem occurs.
func (c *Repair) runOnce() error {
	return retry.RetryOnConflict(retry.DefaultBackoff, c.doRunOnce)
}


doRunOnce

// doRunOnce verifies the state of the port allocations and returns an error if an unrecoverable problem occurs.
func (c *Repair) doRunOnce() error {
	// TODO: (per smarterclayton) if Get() or ListServices() is a weak consistency read,
	// or if they are executed against different leaders,
	// the ordering guarantee required to ensure no port is allocated twice is violated.
	// ListServices must return a ResourceVersion higher than the etcd index Get triggers,
	// and the release code must not release services that have had ports allocated but not yet been created
	// See #8295

	// If etcd server is not running we should wait for some time and fail only then. This is particularly
	// important when we start apiserver and etcd at the same time.
	var snapshot *api.RangeAllocation

	err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
		var err error
		snapshot, err = c.alloc.Get()
		return err == nil, err
	})
	if err != nil {
		return fmt.Errorf("unable to refresh the port allocations: %v", err)
	}
	// If not yet initialized.
	if snapshot.Range == "" {
		snapshot.Range = c.portRange.String()
	}
	// Create an allocator because it is easy to use.
	stored, err := portallocator.NewFromSnapshot(snapshot)
	if err != nil {
		return fmt.Errorf("unable to rebuild allocator from snapshot: %v", err)
	}

	// We explicitly send no resource version, since the resource version
	// of 'snapshot' is from a different collection, it's not comparable to
	// the service collection. The caching layer keeps per-collection RVs,
	// and this is proper, since in theory the collections could be hosted
	// in separate etcd (or even non-etcd) instances.
	list, err := c.serviceClient.Services(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{})
	if err != nil {
		return fmt.Errorf("unable to refresh the port block: %v", err)
	}

	rebuilt, err := portallocator.NewInMemory(c.portRange)
	if err != nil {
		return fmt.Errorf("unable to create port allocator: %v", err)
	}
	// Check every Service's ports, and rebuild the state as we think it should be.
	for i := range list.Items {
		svc := &list.Items[i]
		ports := collectServiceNodePorts(svc)
		if len(ports) == 0 {
			continue
		}

		for _, port := range ports {
			switch err := rebuilt.Allocate(port); err {
			case nil:
				if stored.Has(port) {
					// remove it from the old set, so we can find leaks
					stored.Release(port)
				} else {
					// doesn't seem to be allocated
					c.recorder.Eventf(svc, nil, corev1.EventTypeWarning, "PortNotAllocated", "PortAllocation", "Port %d is not allocated; repairing", port)
					runtime.HandleError(fmt.Errorf("the node port %d for service %s/%s is not allocated; repairing", port, svc.Name, svc.Namespace))
				}
				delete(c.leaks, port) // it is used, so it can't be leaked
			case portallocator.ErrAllocated:
				// port is duplicate, reallocate
				c.recorder.Eventf(svc, nil, corev1.EventTypeWarning, "PortAlreadyAllocated", "PortAllocation", "Port %d was assigned to multiple services; please recreate service", port)
				runtime.HandleError(fmt.Errorf("the node port %d for service %s/%s was assigned to multiple services; please recreate", port, svc.Name, svc.Namespace))
			case err.(*portallocator.ErrNotInRange):
				// port is out of range, reallocate
				c.recorder.Eventf(svc, nil, corev1.EventTypeWarning, "PortOutOfRange", "PortAllocation", "Port %d is not within the port range %s; please recreate service", port, c.portRange)
				runtime.HandleError(fmt.Errorf("the port %d for service %s/%s is not within the port range %s; please recreate", port, svc.Name, svc.Namespace, c.portRange))
			case portallocator.ErrFull:
				// somehow we are out of ports
				c.recorder.Eventf(svc, nil, corev1.EventTypeWarning, "PortRangeFull", "PortAllocation", "Port range %s is full; you must widen the port range in order to create new services", c.portRange)
				return fmt.Errorf("the port range %s is full; you must widen the port range in order to create new services", c.portRange)
			default:
				c.recorder.Eventf(svc, nil, corev1.EventTypeWarning, "UnknownError", "PortAllocation", "Unable to allocate port %d due to an unknown error", port)
				return fmt.Errorf("unable to allocate port %d for service %s/%s due to an unknown error, exiting: %v", port, svc.Name, svc.Namespace, err)
			}
		}
	}

	// Check for ports that are left in the old set.  They appear to have been leaked.
	stored.ForEach(func(port int) {
		count, found := c.leaks[port]
		switch {
		case !found:
			// flag it to be cleaned up after any races (hopefully) are gone
			runtime.HandleError(fmt.Errorf("the node port %d may have leaked: flagging for later clean up", port))
			count = numRepairsBeforeLeakCleanup - 1
			fallthrough
		case count > 0:
			// pretend it is still in use until count expires
			c.leaks[port] = count - 1
			if err := rebuilt.Allocate(port); err != nil {
				runtime.HandleError(fmt.Errorf("the node port %d may have leaked, but can not be allocated: %v", port, err))
			}
		default:
			// do not add it to the rebuilt set, which means it will be available for reuse
			runtime.HandleError(fmt.Errorf("the node port %d appears to have leaked: cleaning up", port))
		}
	})

	// Blast the rebuilt state into storage.
	if err := rebuilt.Snapshot(snapshot); err != nil {
		return fmt.Errorf("unable to snapshot the updated port allocations: %v", err)
	}

	if err := c.alloc.CreateOrUpdate(snapshot); err != nil {
		if errors.IsConflict(err) {
			return err
		}
		return fmt.Errorf("unable to persist the updated port allocations: %v", err)
	}
	return nil
}


collectServiceNodePorts

// collectServiceNodePorts returns nodePorts specified in the Service.
// Please note that:
//  1. same nodePort with *same* protocol will be duplicated as it is
//  2. same nodePort with *different* protocol will be deduplicated
func collectServiceNodePorts(service *corev1.Service) []int {
	var servicePorts []int
	// map from nodePort to set of protocols
	seen := make(map[int]sets.String)
	for _, port := range service.Spec.Ports {
		nodePort := int(port.NodePort)
		if nodePort == 0 {
			continue
		}
		proto := string(port.Protocol)
		s := seen[nodePort]
		if s == nil { // have not seen this nodePort before
			s = sets.NewString(proto)
			servicePorts = append(servicePorts, nodePort)
		} else if s.Has(proto) { // same nodePort with same protocol
			servicePorts = append(servicePorts, nodePort)
		} else { // same nodePort with different protocol
			s.Insert(proto)
		}
		seen[nodePort] = s
	}

	healthPort := int(service.Spec.HealthCheckNodePort)
	if healthPort != 0 {
		s := seen[healthPort]
		// TODO: is it safe to assume the protocol is always TCP?
		if s == nil || s.Has(string(corev1.ProtocolTCP)) {
			servicePorts = append(servicePorts, healthPort)
		}
	}

	return servicePorts
}


registerMetrics

func registerMetrics() {
	registerMetricsOnce.Do(func() {
		legacyregistry.MustRegister(nodePortAllocated)
		legacyregistry.MustRegister(nodePortAvailable)
		legacyregistry.MustRegister(nodePortAllocations)
		legacyregistry.MustRegister(nodePortAllocationErrors)
	})
}


setAllocated

func (m *metricsRecorder) setAllocated(allocated int) {
	nodePortAllocated.Set(float64(allocated))
}


setAvailable

func (m *metricsRecorder) setAvailable(available int) {
	nodePortAvailable.Set(float64(available))
}


incrementAllocations

func (m *metricsRecorder) incrementAllocations(scope string) {
	nodePortAllocations.WithLabelValues(scope).Inc()
}


incrementAllocationErrors

func (m *metricsRecorder) incrementAllocationErrors(scope string) {
	nodePortAllocationErrors.WithLabelValues(scope).Inc()
}


setAllocated

func (*emptyMetricsRecorder) setAllocated(allocated int)             {}


setAvailable

func (*emptyMetricsRecorder) setAvailable(available int)             {}


incrementAllocations

func (*emptyMetricsRecorder) incrementAllocations(scope string)      {}


incrementAllocationErrors

func (*emptyMetricsRecorder) incrementAllocationErrors(scope string) {}


StartOperation

// Creates a portAllocationOperation, tracking a set of allocations & releases
// If dryRun is specified, never actually allocate or release anything
func StartOperation(pa Interface, dryRun bool) *PortAllocationOperation {
	op := &PortAllocationOperation{}
	op.pa = pa
	op.allocated = []int{}
	op.releaseDeferred = []int{}
	op.shouldRollback = true
	op.dryRun = dryRun
	return op
}


Finish

// Will rollback unless marked as shouldRollback = false by a Commit().  Call from a defer block
func (op *PortAllocationOperation) Finish() {
	if op.shouldRollback {
		op.Rollback()
	}
}


Rollback

// (Try to) undo any operations we did
func (op *PortAllocationOperation) Rollback() []error {
	if op.dryRun {
		return nil
	}

	errors := []error{}

	for _, allocated := range op.allocated {
		err := op.pa.Release(allocated)
		if err != nil {
			errors = append(errors, err)
		}
	}

	if len(errors) == 0 {
		return nil
	}
	return errors
}


Commit

// (Try to) perform any deferred operations.
// Note that even if this fails, we don't rollback; we always want to err on the side of over-allocation,
// and Commit should be called _after_ the owner is written
func (op *PortAllocationOperation) Commit() []error {
	if op.dryRun {
		return nil
	}

	errors := []error{}

	for _, release := range op.releaseDeferred {
		err := op.pa.Release(release)
		if err != nil {
			errors = append(errors, err)
		}
	}

	// Even on error, we don't rollback
	// Problems should be fixed by an eventual reconciliation / restart
	op.shouldRollback = false

	if len(errors) == 0 {
		return nil
	}

	return errors
}


Allocate

// Allocates a port, and record it for future rollback
func (op *PortAllocationOperation) Allocate(port int) error {
	if op.dryRun {
		if op.pa.Has(port) {
			return ErrAllocated
		}
		for _, a := range op.allocated {
			if port == a {
				return ErrAllocated
			}
		}
		op.allocated = append(op.allocated, port)
		return nil
	}

	err := op.pa.Allocate(port)
	if err == nil {
		op.allocated = append(op.allocated, port)
	}
	return err
}


AllocateNext

// Allocates a port, and record it for future rollback
func (op *PortAllocationOperation) AllocateNext() (int, error) {
	if op.dryRun {
		// Find the max element of the allocated ports array.
		// If no ports are already being allocated by this operation,
		// then choose a sensible guess for a dummy port number
		var lastPort int
		for _, allocatedPort := range op.allocated {
			if allocatedPort > lastPort {
				lastPort = allocatedPort
			}
		}
		if len(op.allocated) == 0 {
			lastPort = 32768
		}

		// Try to find the next non allocated port.
		// If too many ports are full, just reuse one,
		// since this is just a dummy value.
		for port := lastPort + 1; port < 100; port++ {
			err := op.Allocate(port)
			if err == nil {
				return port, nil
			}
		}
		op.allocated = append(op.allocated, lastPort+1)
		return lastPort + 1, nil
	}

	port, err := op.pa.AllocateNext()
	if err == nil {
		op.allocated = append(op.allocated, port)
	}
	return port, err
}


ReleaseDeferred

// Marks a port so that it will be released if this operation Commits
func (op *PortAllocationOperation) ReleaseDeferred(port int) {
	op.releaseDeferred = append(op.releaseDeferred, port)
}


New

// New returns an empty service resource
func (r *ProxyREST) New() runtime.Object {
	return &api.ServiceProxyOptions{}
}


Destroy

// Destroy cleans up resources on shutdown.
func (r *ProxyREST) Destroy() {
	// Given no underlying store, we don't destroy anything
	// here explicitly.
}


ConnectMethods

// ConnectMethods returns the list of HTTP methods that can be proxied
func (r *ProxyREST) ConnectMethods() []string {
	return proxyMethods
}


NewConnectOptions

// NewConnectOptions returns versioned resource that represents proxy parameters
func (r *ProxyREST) NewConnectOptions() (runtime.Object, bool, string) {
	return &api.ServiceProxyOptions{}, true, "path"
}


Connect

// Connect returns a handler for the service proxy
func (r *ProxyREST) Connect(ctx context.Context, id string, opts runtime.Object, responder rest.Responder) (http.Handler, error) {
	proxyOpts, ok := opts.(*api.ServiceProxyOptions)
	if !ok {
		return nil, fmt.Errorf("Invalid options object: %#v", opts)
	}
	location, transport, err := r.Redirector.ResourceLocation(ctx, id)
	if err != nil {
		return nil, err
	}
	location.Path = net.JoinPreservingTrailingSlash(location.Path, proxyOpts.Path)
	// Return a proxy handler that uses the desired transport, wrapped with additional proxy handling (to get URL rewriting, X-Forwarded-* headers, etc)
	return newThrottledUpgradeAwareProxyHandler(location, transport, true, false, responder), nil
}


newThrottledUpgradeAwareProxyHandler

func newThrottledUpgradeAwareProxyHandler(location *url.URL, transport http.RoundTripper, wrapTransport, upgradeRequired bool, responder rest.Responder) *proxy.UpgradeAwareHandler {
	handler := proxy.NewUpgradeAwareHandler(location, transport, wrapTransport, upgradeRequired, proxy.NewErrorResponder(responder))
	handler.MaxBytesPerSec = capabilities.Get().PerConnectionBandwidthLimitBytesPerSec
	return handler
}


makeAlloc

// This is a trasitionary function to facilitate service REST flattening.
func makeAlloc(defaultFamily api.IPFamily, ipAllocs map[api.IPFamily]ipallocator.Interface, portAlloc portallocator.Interface) Allocators {
	return Allocators{
		defaultServiceIPFamily:      defaultFamily,
		serviceIPAllocatorsByFamily: ipAllocs,
		serviceNodePorts:            portAlloc,
	}
}


allocateCreate

func (al *Allocators) allocateCreate(service *api.Service, dryRun bool) (transaction, error) {
	result := metaTransaction{}
	success := false

	defer func() {
		if !success {
			result.Revert()
		}
	}()

	// Ensure IP family fields are correctly initialized.  We do it here, since
	// we want this to be visible even when dryRun == true.
	if err := al.initIPFamilyFields(After{service}, Before{nil}); err != nil {
		return nil, err
	}

	// Allocate ClusterIPs
	//TODO(thockin): validation should not pass with empty clusterIP, but it
	//does (and is tested!).  Fixing that all is a big PR and will have to
	//happen later.
	if txn, err := al.txnAllocClusterIPs(service, dryRun); err != nil {
		return nil, err
	} else {
		result = append(result, txn)
	}

	// Allocate ports
	if txn, err := al.txnAllocNodePorts(service, dryRun); err != nil {
		return nil, err
	} else {
		result = append(result, txn)
	}

	success = true
	return result, nil
}


initIPFamilyFields

// attempts to default service ip families according to cluster configuration
// while ensuring that provided families are configured on cluster.
func (al *Allocators) initIPFamilyFields(after After, before Before) error {
	oldService, service := before.Service, after.Service

	// can not do anything here
	if service.Spec.Type == api.ServiceTypeExternalName {
		return nil
	}

	// We don't want to auto-upgrade (add an IP) or downgrade (remove an IP)
	// PreferDualStack services following a cluster change to/from
	// dual-stackness.
	//
	// That means a PreferDualStack service will only be upgraded/downgraded
	// when:
	// - changing ipFamilyPolicy to "RequireDualStack" or "SingleStack" AND
	// - adding or removing a secondary clusterIP or ipFamily
	if isMatchingPreferDualStackClusterIPFields(after, before) {
		return nil // nothing more to do.
	}

	// If the user didn't specify ipFamilyPolicy, we can infer a default.  We
	// don't want a static default because we want to make sure that we never
	// change between single- and dual-stack modes with explicit direction, as
	// provided by ipFamilyPolicy.  Consider these cases:
	//   * Create (POST): If they didn't specify a policy we can assume it's
	//     always SingleStack.
	//   * Update (PUT): If they didn't specify a policy we need to adopt the
	//     policy from before.  This is better than always assuming SingleStack
	//     because a PUT that changes clusterIPs from 2 to 1 value but doesn't
	//     specify ipFamily would work.
	//   * Update (PATCH): If they didn't specify a policy it will adopt the
	//     policy from before.
	if service.Spec.IPFamilyPolicy == nil {
		if oldService != nil && oldService.Spec.IPFamilyPolicy != nil {
			// Update from an object with policy, use the old policy
			service.Spec.IPFamilyPolicy = oldService.Spec.IPFamilyPolicy
		} else if service.Spec.ClusterIP == api.ClusterIPNone && len(service.Spec.Selector) == 0 {
			// Special-case: headless + selectorless defaults to dual.
			requireDualStack := api.IPFamilyPolicyRequireDualStack
			service.Spec.IPFamilyPolicy = &requireDualStack
		} else {
			// create or update from an object without policy (e.g.
			// ExternalName) to one that needs policy
			singleStack := api.IPFamilyPolicySingleStack
			service.Spec.IPFamilyPolicy = &singleStack
		}
	}
	// Henceforth we can assume ipFamilyPolicy is set.

	// Do some loose pre-validation of the input.  This makes it easier in the
	// rest of allocation code to not have to consider corner cases.
	// TODO(thockin): when we tighten validation (e.g. to require IPs) we will
	// need a "strict" and a "loose" form of this.
	if el := validation.ValidateServiceClusterIPsRelatedFields(service); len(el) != 0 {
		return errors.NewInvalid(api.Kind("Service"), service.Name, el)
	}

	//TODO(thockin): Move this logic to validation?
	el := make(field.ErrorList, 0)

	// Update-only prep work.
	if oldService != nil {
		if getIPFamilyPolicy(service) == api.IPFamilyPolicySingleStack {
			// As long as ClusterIPs and IPFamilies have not changed, setting
			// the policy to single-stack is clear intent.
			// ClusterIPs[0] is immutable, so it is safe to keep.
			if sameClusterIPs(oldService, service) && len(service.Spec.ClusterIPs) > 1 {
				service.Spec.ClusterIPs = service.Spec.ClusterIPs[0:1]
			}
			if sameIPFamilies(oldService, service) && len(service.Spec.IPFamilies) > 1 {
				service.Spec.IPFamilies = service.Spec.IPFamilies[0:1]
			}
		} else {
			// If the policy is anything but single-stack AND they reduced these
			// fields, it's an error.  They need to specify policy.
			if reducedClusterIPs(After{service}, Before{oldService}) {
				el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy,
					"must be 'SingleStack' to release the secondary cluster IP"))
			}
			if reducedIPFamilies(After{service}, Before{oldService}) {
				el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy,
					"must be 'SingleStack' to release the secondary IP family"))
			}
		}
	}

	// Make sure ipFamilyPolicy makes sense for the provided ipFamilies and
	// clusterIPs.  Further checks happen below - after the special cases.
	if getIPFamilyPolicy(service) == api.IPFamilyPolicySingleStack {
		if len(service.Spec.ClusterIPs) == 2 {
			el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy,
				"must be 'RequireDualStack' or 'PreferDualStack' when multiple cluster IPs are specified"))
		}
		if len(service.Spec.IPFamilies) == 2 {
			el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy,
				"must be 'RequireDualStack' or 'PreferDualStack' when multiple IP families are specified"))
		}
	}

	// Infer IPFamilies[] from ClusterIPs[].  Further checks happen below,
	// after the special cases.
	for i, ip := range service.Spec.ClusterIPs {
		if ip == api.ClusterIPNone {
			break
		}

		// We previously validated that IPs are well-formed and that if an
		// ipFamilies[] entry exists it matches the IP.
		fam := familyOf(ip)

		// If the corresponding family is not specified, add it.
		if i >= len(service.Spec.IPFamilies) {
			// Families are checked more later, but this is a better error in
			// this specific case (indicating the user-provided IP, rather
			// than than the auto-assigned family).
			if _, found := al.serviceIPAllocatorsByFamily[fam]; !found {
				el = append(el, field.Invalid(field.NewPath("spec", "clusterIPs").Index(i), service.Spec.ClusterIPs,
					fmt.Sprintf("%s is not configured on this cluster", fam)))
			} else {
				// OK to infer.
				service.Spec.IPFamilies = append(service.Spec.IPFamilies, fam)
			}
		}
	}

	// If we have validation errors, bail out now so we don't make them worse.
	if len(el) > 0 {
		return errors.NewInvalid(api.Kind("Service"), service.Name, el)
	}

	// Special-case: headless + selectorless.  This has to happen before other
	// checks because it explicitly allows combinations of inputs that would
	// otherwise be errors.
	if service.Spec.ClusterIP == api.ClusterIPNone && len(service.Spec.Selector) == 0 {
		// If IPFamilies was not set by the user, start with the default
		// family.
		if len(service.Spec.IPFamilies) == 0 {
			service.Spec.IPFamilies = []api.IPFamily{al.defaultServiceIPFamily}
		}

		// this follows headful services. With one exception on a single stack
		// cluster the user is allowed to create headless services that has multi families
		// the validation allows it
		if len(service.Spec.IPFamilies) < 2 {
			if *(service.Spec.IPFamilyPolicy) != api.IPFamilyPolicySingleStack {
				// add the alt ipfamily
				if service.Spec.IPFamilies[0] == api.IPv4Protocol {
					service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv6Protocol)
				} else {
					service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv4Protocol)
				}
			}
		}

		// nothing more needed here
		return nil
	}

	//
	// Everything below this MUST happen *after* the above special cases.
	//

	// Demanding dual-stack on a non dual-stack cluster.
	if getIPFamilyPolicy(service) == api.IPFamilyPolicyRequireDualStack {
		if len(al.serviceIPAllocatorsByFamily) < 2 {
			el = append(el, field.Invalid(field.NewPath("spec", "ipFamilyPolicy"), service.Spec.IPFamilyPolicy,
				"this cluster is not configured for dual-stack services"))
		}
	}

	// If there is a family requested then it has to be configured on cluster.
	for i, ipFamily := range service.Spec.IPFamilies {
		if _, found := al.serviceIPAllocatorsByFamily[ipFamily]; !found {
			el = append(el, field.Invalid(field.NewPath("spec", "ipFamilies").Index(i), ipFamily, "not configured on this cluster"))
		}
	}

	// If we have validation errors, don't bother with the rest.
	if len(el) > 0 {
		return errors.NewInvalid(api.Kind("Service"), service.Name, el)
	}

	// nil families, gets cluster default
	if len(service.Spec.IPFamilies) == 0 {
		service.Spec.IPFamilies = []api.IPFamily{al.defaultServiceIPFamily}
	}

	// If this service is looking for dual-stack and this cluster does have two
	// families, append the missing family.
	if *(service.Spec.IPFamilyPolicy) != api.IPFamilyPolicySingleStack &&
		len(service.Spec.IPFamilies) == 1 &&
		len(al.serviceIPAllocatorsByFamily) == 2 {

		if service.Spec.IPFamilies[0] == api.IPv4Protocol {
			service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv6Protocol)
		} else if service.Spec.IPFamilies[0] == api.IPv6Protocol {
			service.Spec.IPFamilies = append(service.Spec.IPFamilies, api.IPv4Protocol)
		}
	}

	return nil
}


txnAllocClusterIPs

func (al *Allocators) txnAllocClusterIPs(service *api.Service, dryRun bool) (transaction, error) {
	// clusterIPs that were allocated may need to be released in case of
	// failure at a higher level.
	allocated, err := al.allocClusterIPs(service, dryRun)
	if err != nil {
		return nil, err
	}

	txn := callbackTransaction{
		revert: func() {
			if dryRun {
				return
			}
			actuallyReleased, err := al.releaseIPs(allocated)
			if err != nil {
				klog.ErrorS(err, "failed to clean up after failed service create",
					"service", klog.KObj(service),
					"shouldRelease", allocated,
					"released", actuallyReleased)
			}
		},
		commit: func() {
			if !dryRun {
				if len(allocated) > 0 {
					klog.InfoS("allocated clusterIPs",
						"service", klog.KObj(service),
						"clusterIPs", allocated)
				}
			}
		},
	}
	return txn, nil
}


allocClusterIPs

// allocates ClusterIPs for a service
func (al *Allocators) allocClusterIPs(service *api.Service, dryRun bool) (map[api.IPFamily]string, error) {
	// external name don't get ClusterIPs
	if service.Spec.Type == api.ServiceTypeExternalName {
		return nil, nil
	}

	// headless don't get ClusterIPs
	if len(service.Spec.ClusterIPs) > 0 && service.Spec.ClusterIPs[0] == api.ClusterIPNone {
		return nil, nil
	}

	toAlloc := make(map[api.IPFamily]string)
	// at this stage, the only fact we know is that service has correct ip families
	// assigned to it. It may have partial assigned ClusterIPs (Upgrade to dual stack)
	// may have no ips at all. The below loop is meant to fix this
	// (we also know that this cluster has these families)

	// if there is no slice to work with
	if service.Spec.ClusterIPs == nil {
		service.Spec.ClusterIPs = make([]string, 0, len(service.Spec.IPFamilies))
	}

	for i, ipFamily := range service.Spec.IPFamilies {
		if i > (len(service.Spec.ClusterIPs) - 1) {
			service.Spec.ClusterIPs = append(service.Spec.ClusterIPs, "" /* just a marker */)
		}

		toAlloc[ipFamily] = service.Spec.ClusterIPs[i]
	}

	// allocate
	allocated, err := al.allocIPs(service, toAlloc, dryRun)

	// set if successful
	if err == nil {
		for family, ip := range allocated {
			for i, check := range service.Spec.IPFamilies {
				if family == check {
					service.Spec.ClusterIPs[i] = ip
					// while we technically don't need to do that testing rest does not
					// go through conversion logic but goes through validation *sigh*.
					// so we set ClusterIP here as well
					// because the testing code expects valid (as they are output-ed from conversion)
					// as it patches fields
					if i == 0 {
						service.Spec.ClusterIP = ip
					}
				}
			}
		}
	}

	return allocated, err
}


allocIPs

func (al *Allocators) allocIPs(service *api.Service, toAlloc map[api.IPFamily]string, dryRun bool) (map[api.IPFamily]string, error) {
	allocated := make(map[api.IPFamily]string)

	for family, ip := range toAlloc {
		allocator := al.serviceIPAllocatorsByFamily[family] // should always be there, as we pre validate
		if dryRun {
			allocator = allocator.DryRun()
		}
		if ip == "" {
			var allocatedIP net.IP
			var err error
			if utilfeature.DefaultFeatureGate.Enabled(features.MultiCIDRServiceAllocator) {
				svcAllocator, ok := allocator.(*ipallocator.Allocator)
				if ok {
					allocatedIP, err = svcAllocator.AllocateNextService(service)
				} else {
					allocatedIP, err = allocator.AllocateNext()
				}
			} else {
				allocatedIP, err = allocator.AllocateNext()
			}
			if err != nil {
				return allocated, errors.NewInternalError(fmt.Errorf("failed to allocate a serviceIP: %v", err))
			}
			allocated[family] = allocatedIP.String()
		} else {
			parsedIP := netutils.ParseIPSloppy(ip)
			if parsedIP == nil {
				return allocated, errors.NewInternalError(fmt.Errorf("failed to parse service IP %q", ip))
			}
			var err error
			if utilfeature.DefaultFeatureGate.Enabled(features.MultiCIDRServiceAllocator) {
				svcAllocator, ok := allocator.(*ipallocator.Allocator)
				if ok {
					err = svcAllocator.AllocateService(service, parsedIP)
				} else {
					err = allocator.Allocate(parsedIP)
				}
			} else {
				err = allocator.Allocate(parsedIP)
			}
			if err != nil {
				el := field.ErrorList{field.Invalid(field.NewPath("spec", "clusterIPs"), service.Spec.ClusterIPs, fmt.Sprintf("failed to allocate IP %v: %v", ip, err))}
				return allocated, errors.NewInvalid(api.Kind("Service"), service.Name, el)
			}
			allocated[family] = ip
		}
	}
	return allocated, nil
}


releaseIPs

// releases clusterIPs per family
func (al *Allocators) releaseIPs(toRelease map[api.IPFamily]string) (map[api.IPFamily]string, error) {
	if toRelease == nil {
		return nil, nil
	}

	released := make(map[api.IPFamily]string)
	for family, ip := range toRelease {
		allocator, ok := al.serviceIPAllocatorsByFamily[family]
		if !ok {
			// Maybe the cluster was previously configured for dual-stack,
			// then switched to single-stack?
			klog.InfoS("Not releasing ClusterIP because related family is not enabled", "clusterIP", ip, "family", family)
			continue
		}

		parsedIP := netutils.ParseIPSloppy(ip)
		if parsedIP == nil {
			return released, errors.NewInternalError(fmt.Errorf("failed to parse service IP %q", ip))
		}
		if err := allocator.Release(parsedIP); err != nil {
			return released, err
		}
		released[family] = ip
	}

	return released, nil
}


txnAllocNodePorts

func (al *Allocators) txnAllocNodePorts(service *api.Service, dryRun bool) (transaction, error) {
	// The allocator tracks dry-run-ness internally.
	nodePortOp := portallocator.StartOperation(al.serviceNodePorts, dryRun)

	txn := callbackTransaction{
		commit: func() {
			nodePortOp.Commit()
			// We don't NEED to call Finish() here, but for that package says
			// to, so for future-safety, we will.
			nodePortOp.Finish()
		},
		revert: func() {
			// Weirdly named but this will revert if commit wasn't called
			nodePortOp.Finish()
		},
	}

	// Allocate NodePorts, if needed.
	if service.Spec.Type == api.ServiceTypeNodePort || service.Spec.Type == api.ServiceTypeLoadBalancer {
		if err := initNodePorts(service, nodePortOp); err != nil {
			txn.Revert()
			return nil, err
		}
	}

	// Handle ExternalTraffic related fields during service creation.
	if apiservice.NeedsHealthCheck(service) {
		if err := al.allocHealthCheckNodePort(service, nodePortOp); err != nil {
			txn.Revert()
			return nil, errors.NewInternalError(err)
		}
	}

	return txn, nil
}


initNodePorts

func initNodePorts(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) error {
	svcPortToNodePort := map[int]int{}
	for i := range service.Spec.Ports {
		servicePort := &service.Spec.Ports[i]
		if servicePort.NodePort == 0 && !shouldAllocateNodePorts(service) {
			// Don't allocate new ports, but do respect specific requests.
			continue
		}
		allocatedNodePort := svcPortToNodePort[int(servicePort.Port)]
		if allocatedNodePort == 0 {
			// This will only scan forward in the service.Spec.Ports list because any matches
			// before the current port would have been found in svcPortToNodePort. This is really
			// looking for any user provided values.
			np := findRequestedNodePort(int(servicePort.Port), service.Spec.Ports)
			if np != 0 {
				err := nodePortOp.Allocate(np)
				if err != nil {
					// TODO: when validation becomes versioned, this gets more complicated.
					el := field.ErrorList{field.Invalid(field.NewPath("spec", "ports").Index(i).Child("nodePort"), np, err.Error())}
					return errors.NewInvalid(api.Kind("Service"), service.Name, el)
				}
				servicePort.NodePort = int32(np)
				svcPortToNodePort[int(servicePort.Port)] = np
			} else {
				nodePort, err := nodePortOp.AllocateNext()
				if err != nil {
					// TODO: what error should be returned here?  It's not a
					// field-level validation failure (the field is valid), and it's
					// not really an internal error.
					return errors.NewInternalError(fmt.Errorf("failed to allocate a nodePort: %v", err))
				}
				servicePort.NodePort = int32(nodePort)
				svcPortToNodePort[int(servicePort.Port)] = nodePort
			}
		} else if int(servicePort.NodePort) != allocatedNodePort {
			// TODO(xiangpengzhao): do we need to allocate a new NodePort in this case?
			// Note: the current implementation is better, because it saves a NodePort.
			if servicePort.NodePort == 0 {
				servicePort.NodePort = int32(allocatedNodePort)
			} else {
				err := nodePortOp.Allocate(int(servicePort.NodePort))
				if err != nil {
					// TODO: when validation becomes versioned, this gets more complicated.
					el := field.ErrorList{field.Invalid(field.NewPath("spec", "ports").Index(i).Child("nodePort"), servicePort.NodePort, err.Error())}
					return errors.NewInvalid(api.Kind("Service"), service.Name, el)
				}
			}
		}
	}

	return nil
}


allocHealthCheckNodePort

// allocHealthCheckNodePort allocates health check node port to service.
func (al *Allocators) allocHealthCheckNodePort(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) error {
	healthCheckNodePort := service.Spec.HealthCheckNodePort
	if healthCheckNodePort != 0 {
		// If the request has a health check nodePort in mind, attempt to reserve it.
		err := nodePortOp.Allocate(int(healthCheckNodePort))
		if err != nil {
			return fmt.Errorf("failed to allocate requested HealthCheck NodePort %v: %v",
				healthCheckNodePort, err)
		}
	} else {
		// If the request has no health check nodePort specified, allocate any.
		healthCheckNodePort, err := nodePortOp.AllocateNext()
		if err != nil {
			return fmt.Errorf("failed to allocate a HealthCheck NodePort %v: %v", healthCheckNodePort, err)
		}
		service.Spec.HealthCheckNodePort = int32(healthCheckNodePort)
	}
	return nil
}


allocateUpdate

func (al *Allocators) allocateUpdate(after After, before Before, dryRun bool) (transaction, error) {
	result := metaTransaction{}
	success := false

	defer func() {
		if !success {
			result.Revert()
		}
	}()

	// Ensure IP family fields are correctly initialized.  We do it here, since
	// we want this to be visible even when dryRun == true.
	if err := al.initIPFamilyFields(after, before); err != nil {
		return nil, err
	}

	// Allocate ClusterIPs
	//TODO(thockin): validation should not pass with empty clusterIP, but it
	//does (and is tested!).  Fixing that all is a big PR and will have to
	//happen later.
	if txn, err := al.txnUpdateClusterIPs(after, before, dryRun); err != nil {
		return nil, err
	} else {
		result = append(result, txn)
	}

	// Allocate ports
	if txn, err := al.txnUpdateNodePorts(after, before, dryRun); err != nil {
		return nil, err
	} else {
		result = append(result, txn)
	}

	success = true
	return result, nil
}


txnUpdateClusterIPs

func (al *Allocators) txnUpdateClusterIPs(after After, before Before, dryRun bool) (transaction, error) {
	service := after.Service

	allocated, released, err := al.updateClusterIPs(after, before, dryRun)
	if err != nil {
		return nil, err
	}

	// on failure: Any newly allocated IP must be released back
	// on failure: Any previously allocated IP that would have been released,
	//             must *not* be released
	// on success: Any previously allocated IP that should be released, will be
	//             released
	txn := callbackTransaction{
		commit: func() {
			if dryRun {
				return
			}
			if len(allocated) > 0 {
				klog.InfoS("allocated clusterIPs",
					"service", klog.KObj(service),
					"clusterIPs", allocated)
			}
			if actuallyReleased, err := al.releaseIPs(released); err != nil {
				klog.ErrorS(err, "failed to clean up after successful service update",
					"service", klog.KObj(service),
					"shouldRelease", released,
					"released", actuallyReleased)
			}
		},
		revert: func() {
			if dryRun {
				return
			}
			if actuallyReleased, err := al.releaseIPs(allocated); err != nil {
				klog.ErrorS(err, "failed to clean up after failed service update",
					"service", klog.KObj(service),
					"shouldRelease", allocated,
					"released", actuallyReleased)
			}
		},
	}
	return txn, nil
}


updateClusterIPs

// handles type change/upgrade/downgrade change type for an update service
// this func does not perform actual release of clusterIPs. it returns
// a map[family]ip for the caller to release when everything else has
// executed successfully
func (al *Allocators) updateClusterIPs(after After, before Before, dryRun bool) (allocated map[api.IPFamily]string, toRelease map[api.IPFamily]string, err error) {
	oldService, service := before.Service, after.Service

	// We don't want to auto-upgrade (add an IP) or downgrade (remove an IP)
	// PreferDualStack services following a cluster change to/from
	// dual-stackness.
	//
	// That means a PreferDualStack service will only be upgraded/downgraded
	// when:
	// - changing ipFamilyPolicy to "RequireDualStack" or "SingleStack" AND
	// - adding or removing a secondary clusterIP or ipFamily
	if isMatchingPreferDualStackClusterIPFields(after, before) {
		return allocated, toRelease, nil // nothing more to do.
	}

	// use cases:
	// A: service changing types from ExternalName TO ClusterIP types ==> allocate all new
	// B: service changing types from ClusterIP types TO ExternalName ==> release all allocated
	// C: Service upgrading to dual stack  ==> partial allocation
	// D: service downgrading from dual stack ==> partial release

	// CASE A:
	// Update service from ExternalName to non-ExternalName, should initialize ClusterIP.
	if oldService.Spec.Type == api.ServiceTypeExternalName && service.Spec.Type != api.ServiceTypeExternalName {
		allocated, err := al.allocClusterIPs(service, dryRun)
		return allocated, nil, err
	}

	// if headless service then we bail out early (no clusterIPs management needed)
	if len(oldService.Spec.ClusterIPs) > 0 && oldService.Spec.ClusterIPs[0] == api.ClusterIPNone {
		return nil, nil, nil
	}

	// CASE B:
	// Update service from non-ExternalName to ExternalName, should release ClusterIP if exists.
	if oldService.Spec.Type != api.ServiceTypeExternalName && service.Spec.Type == api.ServiceTypeExternalName {
		toRelease = make(map[api.IPFamily]string)
		for i, family := range oldService.Spec.IPFamilies {
			toRelease[family] = oldService.Spec.ClusterIPs[i]
		}
		return nil, toRelease, nil
	}

	upgraded := len(oldService.Spec.IPFamilies) == 1 && len(service.Spec.IPFamilies) == 2
	downgraded := len(oldService.Spec.IPFamilies) == 2 && len(service.Spec.IPFamilies) == 1

	// CASE C:
	if upgraded {
		toAllocate := make(map[api.IPFamily]string)
		// if secondary ip was named, just get it. if not add a marker
		if len(service.Spec.ClusterIPs) < 2 {
			service.Spec.ClusterIPs = append(service.Spec.ClusterIPs, "" /* marker */)
		}

		toAllocate[service.Spec.IPFamilies[1]] = service.Spec.ClusterIPs[1]

		// allocate
		allocated, err := al.allocIPs(service, toAllocate, dryRun)
		// set if successful
		if err == nil {
			service.Spec.ClusterIPs[1] = allocated[service.Spec.IPFamilies[1]]
		}

		return allocated, nil, err
	}

	// CASE D:
	if downgraded {
		toRelease = make(map[api.IPFamily]string)
		toRelease[oldService.Spec.IPFamilies[1]] = oldService.Spec.ClusterIPs[1]
		// note: we don't release clusterIP, this is left to clean up in the action itself
		return nil, toRelease, err
	}
	// it was not an upgrade nor downgrade
	return nil, nil, nil
}


txnUpdateNodePorts

func (al *Allocators) txnUpdateNodePorts(after After, before Before, dryRun bool) (transaction, error) {
	oldService, service := before.Service, after.Service

	// The allocator tracks dry-run-ness internally.
	nodePortOp := portallocator.StartOperation(al.serviceNodePorts, dryRun)

	txn := callbackTransaction{
		commit: func() {
			nodePortOp.Commit()
			// We don't NEED to call Finish() here, but for that package says
			// to, so for future-safety, we will.
			nodePortOp.Finish()
		},
		revert: func() {
			// Weirdly named but this will revert if commit wasn't called
			nodePortOp.Finish()
		},
	}

	// Update service from NodePort or LoadBalancer to ExternalName or ClusterIP, should release NodePort if exists.
	if (oldService.Spec.Type == api.ServiceTypeNodePort || oldService.Spec.Type == api.ServiceTypeLoadBalancer) &&
		(service.Spec.Type == api.ServiceTypeExternalName || service.Spec.Type == api.ServiceTypeClusterIP) {
		al.releaseNodePorts(oldService, nodePortOp)
	}

	// Update service from any type to NodePort or LoadBalancer, should update NodePort.
	if service.Spec.Type == api.ServiceTypeNodePort || service.Spec.Type == api.ServiceTypeLoadBalancer {
		if err := al.updateNodePorts(After{service}, Before{oldService}, nodePortOp); err != nil {
			txn.Revert()
			return nil, err
		}
	}

	// Handle ExternalTraffic related updates.
	success, err := al.updateHealthCheckNodePort(After{service}, Before{oldService}, nodePortOp)
	if !success || err != nil {
		txn.Revert()
		return nil, err
	}

	return txn, nil
}


releaseNodePorts

func (al *Allocators) releaseNodePorts(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) {
	nodePorts := collectServiceNodePorts(service)

	for _, nodePort := range nodePorts {
		nodePortOp.ReleaseDeferred(nodePort)
	}
}


updateNodePorts

func (al *Allocators) updateNodePorts(after After, before Before, nodePortOp *portallocator.PortAllocationOperation) error {
	oldService, newService := before.Service, after.Service

	oldNodePortsNumbers := collectServiceNodePorts(oldService)
	newNodePorts := []ServiceNodePort{}
	portAllocated := map[int]bool{}

	for i := range newService.Spec.Ports {
		servicePort := &newService.Spec.Ports[i]
		if servicePort.NodePort == 0 && !shouldAllocateNodePorts(newService) {
			// Don't allocate new ports, but do respect specific requests.
			continue
		}
		nodePort := ServiceNodePort{Protocol: servicePort.Protocol, NodePort: servicePort.NodePort}
		if nodePort.NodePort != 0 {
			if !containsNumber(oldNodePortsNumbers, int(nodePort.NodePort)) && !portAllocated[int(nodePort.NodePort)] {
				err := nodePortOp.Allocate(int(nodePort.NodePort))
				if err != nil {
					el := field.ErrorList{field.Invalid(field.NewPath("spec", "ports").Index(i).Child("nodePort"), nodePort.NodePort, err.Error())}
					return errors.NewInvalid(api.Kind("Service"), newService.Name, el)
				}
				portAllocated[int(nodePort.NodePort)] = true
			}
		} else {
			nodePortNumber, err := nodePortOp.AllocateNext()
			if err != nil {
				// TODO: what error should be returned here?  It's not a
				// field-level validation failure (the field is valid), and it's
				// not really an internal error.
				return errors.NewInternalError(fmt.Errorf("failed to allocate a nodePort: %v", err))
			}
			servicePort.NodePort = int32(nodePortNumber)
			nodePort.NodePort = servicePort.NodePort
		}
		if containsNodePort(newNodePorts, nodePort) {
			return fmt.Errorf("duplicate nodePort: %v", nodePort)
		}
		newNodePorts = append(newNodePorts, nodePort)
	}

	newNodePortsNumbers := collectServiceNodePorts(newService)

	// The comparison loops are O(N^2), but we don't expect N to be huge
	// (there's a hard-limit at 2^16, because they're ports; and even 4 ports would be a lot)
	for _, oldNodePortNumber := range oldNodePortsNumbers {
		if containsNumber(newNodePortsNumbers, oldNodePortNumber) {
			continue
		}
		nodePortOp.ReleaseDeferred(int(oldNodePortNumber))
	}

	return nil
}