// composeMigVolID composes migration handle for RBD PV
// mig_mons-afcca55bc1bdd3f479be1e8281c13ab1_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_7265706c696361706f6f6c
func composeMigVolID(mons string, pool string, image string) string {
	clusterIDInHandle := md5.Sum([]byte(mons))
	clusterField := monsPfx + fmt.Sprintf("%x", clusterIDInHandle)
	poolHashInHandle := hex.EncodeToString([]byte(pool))
	imageHashInHandle := strings.TrimPrefix(image, defaultIntreeImagePfx)
	imageField := imgPfx + imageHashInHandle
	volHash := strings.Join([]string{migVolPfx, clusterField, imageField, poolHashInHandle}, "_")
	return volHash
}


// fillVolAttrsForRequest fill the volume attributes for node operations
func fillVolAttrsForRequest(pv *v1.PersistentVolume, volumeAttributes map[string]string) error {
	if pv == nil || pv.Spec.RBD == nil {
		return fmt.Errorf("pv is nil or RBD Volume not defined on pv")
	}
	volumeAttributes[imgNameKey] = pv.Spec.RBD.RBDImage
	volumeAttributes[poolKey] = pv.Spec.RBD.RBDPool
	volumeAttributes[imgFeatureKey] = pv.Annotations[imgFeatureKey]
	volumeAttributes[imgFmtKey] = pv.Annotations[imgFmtKey]
	volumeAttributes[journalPoolKey] = pv.Annotations[journalPoolKey]
	volumeAttributes[defaultMigKey] = defaultMigStaticVal
	volumeAttributes["tryOtherMounters"] = defaultMigStaticVal
	return nil
}


// fillAnnotationsFromCSISource capture required information from csi source
func fillAnnotationsFromCSISource(pv *v1.PersistentVolume, csiSource *v1.CSIPersistentVolumeSource) {
	pv.Annotations[CSIRBDVolHandleAnnKey] = csiSource.VolumeHandle
	pv.Annotations[clusterIDKey] = csiSource.VolumeAttributes[clusterIDKey]
	pv.Annotations[journalPoolKey] = csiSource.VolumeAttributes[journalPoolKey]
	pv.Annotations[imgFeatureKey] = csiSource.VolumeAttributes[imgFeatureKey]
	pv.Annotations[imgFmtKey] = csiSource.VolumeAttributes[imgFmtKey]
}


// NewvSphereCSITranslator returns a new instance of vSphereCSITranslator
func NewvSphereCSITranslator() InTreePlugin {
	return &vSphereCSITranslator{}
}


// TranslateInTreeStorageClassToCSI translates InTree vSphere storage class parameters to CSI storage class
func (t *vSphereCSITranslator) TranslateInTreeStorageClassToCSI(sc *storage.StorageClass) (*storage.StorageClass, error) {
	if sc == nil {
		return nil, fmt.Errorf("sc is nil")
	}
	var params = map[string]string{}
	for k, v := range sc.Parameters {
		switch strings.ToLower(k) {
		case fsTypeKey:
			params[csiFsTypeKey] = v
		case paramStoragePolicyName:
			params[paramStoragePolicyName] = v
		case "datastore":
			params[paramDatastore] = v
		case "diskformat":
			params[paramDiskFormat] = v
		case "hostfailurestotolerate":
			params[paramHostFailuresToTolerate] = v
		case "forceprovisioning":
			params[paramForceProvisioning] = v
		case "cachereservation":
			params[paramCacheReservation] = v
		case "diskstripes":
			params[paramDiskstripes] = v
		case "objectspacereservation":
			params[paramObjectspacereservation] = v
		case "iopslimit":
			params[paramIopslimit] = v
		default:
			klog.V(2).Infof("StorageClass parameter [name:%q, value:%q] is not supported", k, v)
		}
	}

	// This helps vSphere CSI driver to identify in-tree provisioner request vs CSI provisioner request
	// When this is true, Driver returns initialvolumefilepath in the VolumeContext, which is
	// used in TranslateCSIPVToInTree
	params[paramcsiMigration] = "true"
	// translate AllowedTopologies to vSphere CSI Driver topology
	if len(sc.AllowedTopologies) > 0 {
		newTopologies, err := translateAllowedTopologies(sc.AllowedTopologies, vSphereCSITopologyZoneKey)
		if err != nil {
			return nil, fmt.Errorf("failed translating allowed topologies: %v", err)
		}
		sc.AllowedTopologies = newTopologies
	}
	sc.Parameters = params
	return sc, nil
}


// TranslateInTreeInlineVolumeToCSI takes a Volume with VsphereVolume set from in-tree
// and converts the VsphereVolume source to a CSIPersistentVolumeSource
func (t *vSphereCSITranslator) TranslateInTreeInlineVolumeToCSI(volume *v1.Volume, podNamespace string) (*v1.PersistentVolume, error) {
	if volume == nil || volume.VsphereVolume == nil {
		return nil, fmt.Errorf("volume is nil or VsphereVolume not defined on volume")
	}
	pv := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			// Must be unique per disk as it is used as the unique part of the
			// staging path
			Name: fmt.Sprintf("%s-%s", VSphereDriverName, volume.VsphereVolume.VolumePath),
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeSource: v1.PersistentVolumeSource{
				CSI: &v1.CSIPersistentVolumeSource{
					Driver:           VSphereDriverName,
					VolumeHandle:     volume.VsphereVolume.VolumePath,
					FSType:           volume.VsphereVolume.FSType,
					VolumeAttributes: make(map[string]string),
				},
			},
			AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
		},
	}
	if volume.VsphereVolume.StoragePolicyName != "" {
		pv.Spec.CSI.VolumeAttributes[paramStoragePolicyName] = pv.Spec.VsphereVolume.StoragePolicyName
	}
	return pv, nil
}


// TranslateInTreePVToCSI takes a PV with VsphereVolume set from in-tree
// and converts the VsphereVolume source to a CSIPersistentVolumeSource
func (t *vSphereCSITranslator) TranslateInTreePVToCSI(pv *v1.PersistentVolume) (*v1.PersistentVolume, error) {
	if pv == nil || pv.Spec.VsphereVolume == nil {
		return nil, fmt.Errorf("pv is nil or VsphereVolume not defined on pv")
	}
	csiSource := &v1.CSIPersistentVolumeSource{
		Driver:           VSphereDriverName,
		VolumeHandle:     pv.Spec.VsphereVolume.VolumePath,
		FSType:           pv.Spec.VsphereVolume.FSType,
		VolumeAttributes: make(map[string]string),
	}
	if pv.Spec.VsphereVolume.StoragePolicyName != "" {
		csiSource.VolumeAttributes[paramStoragePolicyName] = pv.Spec.VsphereVolume.StoragePolicyName
	}
	// translate in-tree topology to CSI topology for migration
	if err := translateTopologyFromInTreevSphereToCSI(pv, vSphereCSITopologyZoneKey, vSphereCSITopologyRegionKey); err != nil {
		return nil, fmt.Errorf("failed to translate topology: %v", err)
	}
	pv.Spec.VsphereVolume = nil
	pv.Spec.CSI = csiSource
	return pv, nil
}


// TranslateCSIPVToInTree takes a PV with CSIPersistentVolumeSource set and
// translates the vSphere CSI source to a vSphereVolume source.
func (t *vSphereCSITranslator) TranslateCSIPVToInTree(pv *v1.PersistentVolume) (*v1.PersistentVolume, error) {
	if pv == nil || pv.Spec.CSI == nil {
		return nil, fmt.Errorf("pv is nil or CSI source not defined on pv")
	}
	csiSource := pv.Spec.CSI
	vsphereVirtualDiskVolumeSource := &v1.VsphereVirtualDiskVolumeSource{
		FSType: csiSource.FSType,
	}
	volumeFilePath, ok := csiSource.VolumeAttributes[AttributeInitialVolumeFilepath]
	if ok {
		vsphereVirtualDiskVolumeSource.VolumePath = volumeFilePath
	}
	// translate CSI topology to In-tree topology for rollback compatibility.
	if err := translateTopologyFromCSIToInTreevSphere(pv, vSphereCSITopologyZoneKey, vSphereCSITopologyRegionKey); err != nil {
		return nil, fmt.Errorf("failed to translate topology. PV:%+v. Error:%v", *pv, err)
	}
	pv.Spec.CSI = nil
	pv.Spec.VsphereVolume = vsphereVirtualDiskVolumeSource
	return pv, nil
}


// CanSupport tests whether the plugin supports a given persistent volume
// specification from the API.
func (t *vSphereCSITranslator) CanSupport(pv *v1.PersistentVolume) bool {
	return pv != nil && pv.Spec.VsphereVolume != nil
}


// CanSupportInline tests whether the plugin supports a given inline volume
// specification from the API.
func (t *vSphereCSITranslator) CanSupportInline(volume *v1.Volume) bool {
	return volume != nil && volume.VsphereVolume != nil
}


// GetInTreePluginName returns the name of the in-tree plugin driver
func (t *vSphereCSITranslator) GetInTreePluginName() string {
	return VSphereInTreePluginName
}


// GetCSIPluginName returns the name of the CSI plugin
func (t *vSphereCSITranslator) GetCSIPluginName() string {
	return VSphereDriverName
}


// RepairVolumeHandle is needed in VerifyVolumesAttached on the external attacher when we need to do strict volume
// handle matching to check VolumeAttachment attached status.
// vSphere volume does not need patch to help verify whether that volume is attached.
func (t *vSphereCSITranslator) RepairVolumeHandle(volumeHandle, nodeID string) (string, error) {
	return volumeHandle, nil
}


// translateTopologyFromInTreevSphereToCSI converts existing zone labels or in-tree vsphere topology to
// vSphere CSI topology.
func translateTopologyFromInTreevSphereToCSI(pv *v1.PersistentVolume, csiTopologyKeyZone string, csiTopologyKeyRegion string) error {
	zoneLabel, regionLabel := getTopologyLabel(pv)

	// If Zone kubernetes topology exist, replace it to use csiTopologyKeyZone
	zones := getTopologyValues(pv, zoneLabel)
	if len(zones) > 0 {
		replaceTopology(pv, zoneLabel, csiTopologyKeyZone)
	} else {
		// if nothing is in the NodeAffinity, try to fetch the topology from PV labels
		if label, ok := pv.Labels[zoneLabel]; ok {
			if len(label) > 0 {
				addTopology(pv, csiTopologyKeyZone, []string{label})
			}
		}
	}

	// If region kubernetes topology exist, replace it to use csiTopologyKeyRegion
	regions := getTopologyValues(pv, regionLabel)
	if len(regions) > 0 {
		replaceTopology(pv, regionLabel, csiTopologyKeyRegion)
	} else {
		// if nothing is in the NodeAffinity, try to fetch the topology from PV labels
		if label, ok := pv.Labels[regionLabel]; ok {
			if len(label) > 0 {
				addTopology(pv, csiTopologyKeyRegion, []string{label})
			}
		}
	}
	return nil
}


// translateTopologyFromCSIToInTreevSphere converts CSI zone/region affinity rules to in-tree vSphere zone/region labels
func translateTopologyFromCSIToInTreevSphere(pv *v1.PersistentVolume,
	csiTopologyKeyZone string, csiTopologyKeyRegion string) error {
	zoneLabel, regionLabel := getTopologyLabel(pv)

	// Replace all CSI topology to Kubernetes Zone label
	err := replaceTopology(pv, csiTopologyKeyZone, zoneLabel)
	if err != nil {
		return fmt.Errorf("failed to replace CSI topology to Kubernetes topology, error: %v", err)
	}

	// Replace all CSI topology to Kubernetes Region label
	err = replaceTopology(pv, csiTopologyKeyRegion, regionLabel)
	if err != nil {
		return fmt.Errorf("failed to replace CSI topology to Kubernetes topology, error: %v", err)
	}

	zoneVals := getTopologyValues(pv, zoneLabel)
	if len(zoneVals) > 0 {
		if pv.Labels == nil {
			pv.Labels = make(map[string]string)
		}
		_, zoneOK := pv.Labels[zoneLabel]
		if !zoneOK {
			pv.Labels[zoneLabel] = zoneVals[0]
		}
	}
	regionVals := getTopologyValues(pv, regionLabel)
	if len(regionVals) > 0 {
		if pv.Labels == nil {
			pv.Labels = make(map[string]string)
		}
		_, regionOK := pv.Labels[regionLabel]
		if !regionOK {
			pv.Labels[regionLabel] = regionVals[0]
		}
	}
	return nil
}


func New() CSITranslator

// New creates a new CSITranslator which does real translation
// for "in-tree plugins <-> csi drivers"
func New() CSITranslator {
	return CSITranslator{}
}


// TranslateInTreeStorageClassToCSI takes in-tree Storage Class
// and translates it to a set of parameters consumable by CSI plugin
func (CSITranslator) TranslateInTreeStorageClassToCSI(inTreePluginName string, sc *storage.StorageClass) (*storage.StorageClass, error) {
	newSC := sc.DeepCopy()
	for _, curPlugin := range inTreePlugins {
		if inTreePluginName == curPlugin.GetInTreePluginName() {
			return curPlugin.TranslateInTreeStorageClassToCSI(newSC)
		}
	}
	return nil, fmt.Errorf("could not find in-tree storage class parameter translation logic for %#v", inTreePluginName)
}


// TranslateInTreeInlineVolumeToCSI takes a inline volume and will translate
// the in-tree volume source to a CSIPersistentVolumeSource (wrapped in a PV)
// if the translation logic has been implemented.
func (CSITranslator) TranslateInTreeInlineVolumeToCSI(volume *v1.Volume, podNamespace string) (*v1.PersistentVolume, error) {
	if volume == nil {
		return nil, fmt.Errorf("persistent volume was nil")
	}
	for _, curPlugin := range inTreePlugins {
		if curPlugin.CanSupportInline(volume) {
			pv, err := curPlugin.TranslateInTreeInlineVolumeToCSI(volume, podNamespace)
			if err != nil {
				return nil, err
			}
			// Inline volumes only support PersistentVolumeFilesystem (and not block).
			// If VolumeMode has not been set explicitly by plugin-specific
			// translator, set it to Filesystem here.
			// This is only necessary for inline volumes as the default PV
			// initialization that populates VolumeMode does not apply to inline volumes.
			if pv.Spec.VolumeMode == nil {
				volumeMode := v1.PersistentVolumeFilesystem
				pv.Spec.VolumeMode = &volumeMode
			}
			return pv, nil
		}
	}
	return nil, fmt.Errorf("could not find in-tree plugin translation logic for %#v", volume.Name)
}


// TranslateInTreePVToCSI takes a persistent volume and will translate
// the in-tree source to a CSI Source if the translation logic
// has been implemented. The input persistent volume will not
// be modified
func (CSITranslator) TranslateInTreePVToCSI(pv *v1.PersistentVolume) (*v1.PersistentVolume, error) {
	if pv == nil {
		return nil, errors.New("persistent volume was nil")
	}
	copiedPV := pv.DeepCopy()
	for _, curPlugin := range inTreePlugins {
		if curPlugin.CanSupport(copiedPV) {
			return curPlugin.TranslateInTreePVToCSI(copiedPV)
		}
	}
	return nil, fmt.Errorf("could not find in-tree plugin translation logic for %#v", copiedPV.Name)
}


// TranslateCSIPVToInTree takes a PV with a CSI PersistentVolume Source and will translate
// it to a in-tree Persistent Volume Source for the specific in-tree volume specified
// by the `Driver` field in the CSI Source. The input PV object will not be modified.
func (CSITranslator) TranslateCSIPVToInTree(pv *v1.PersistentVolume) (*v1.PersistentVolume, error) {
	if pv == nil || pv.Spec.CSI == nil {
		return nil, errors.New("CSI persistent volume was nil")
	}
	copiedPV := pv.DeepCopy()
	for driverName, curPlugin := range inTreePlugins {
		if copiedPV.Spec.CSI.Driver == driverName {
			return curPlugin.TranslateCSIPVToInTree(copiedPV)
		}
	}
	return nil, fmt.Errorf("could not find in-tree plugin translation logic for %s", copiedPV.Spec.CSI.Driver)
}


// IsMigratableIntreePluginByName tests whether there is migration logic for the in-tree plugin
// whose name matches the given name
func (CSITranslator) IsMigratableIntreePluginByName(inTreePluginName string) bool {
	for _, curPlugin := range inTreePlugins {
		if curPlugin.GetInTreePluginName() == inTreePluginName {
			return true
		}
	}
	return false
}


// IsMigratedCSIDriverByName tests whether there exists an in-tree plugin with logic
// to migrate to the CSI driver with given name
func (CSITranslator) IsMigratedCSIDriverByName(csiPluginName string) bool {
	if _, ok := inTreePlugins[csiPluginName]; ok {
		return true
	}
	return false
}


// GetInTreePluginNameFromSpec returns the plugin name
func (CSITranslator) GetInTreePluginNameFromSpec(pv *v1.PersistentVolume, vol *v1.Volume) (string, error) {
	if pv != nil {
		for _, curPlugin := range inTreePlugins {
			if curPlugin.CanSupport(pv) {
				return curPlugin.GetInTreePluginName(), nil
			}
		}
		return "", fmt.Errorf("could not find in-tree plugin name from persistent volume %v", pv)
	} else if vol != nil {
		for _, curPlugin := range inTreePlugins {
			if curPlugin.CanSupportInline(vol) {
				return curPlugin.GetInTreePluginName(), nil
			}
		}
		return "", fmt.Errorf("could not find in-tree plugin name from volume %v", vol)
	} else {
		return "", errors.New("both persistent volume and volume are nil")
	}
}


// GetCSINameFromInTreeName returns the name of a CSI driver that supersedes the
// in-tree plugin with the given name
func (CSITranslator) GetCSINameFromInTreeName(pluginName string) (string, error) {
	for csiDriverName, curPlugin := range inTreePlugins {
		if curPlugin.GetInTreePluginName() == pluginName {
			return csiDriverName, nil
		}
	}
	return "", fmt.Errorf("could not find CSI Driver name for plugin %v", pluginName)
}


// GetInTreeNameFromCSIName returns the name of the in-tree plugin superseded by
// a CSI driver with the given name
func (CSITranslator) GetInTreeNameFromCSIName(pluginName string) (string, error) {
	if plugin, ok := inTreePlugins[pluginName]; ok {
		return plugin.GetInTreePluginName(), nil
	}
	return "", fmt.Errorf("could not find In-Tree driver name for CSI plugin %v", pluginName)
}


// IsPVMigratable tests whether there is migration logic for the given Persistent Volume
func (CSITranslator) IsPVMigratable(pv *v1.PersistentVolume) bool {
	for _, curPlugin := range inTreePlugins {
		if curPlugin.CanSupport(pv) {
			return true
		}
	}
	return false
}


// IsInlineMigratable tests whether there is Migration logic for the given Inline Volume
func (CSITranslator) IsInlineMigratable(vol *v1.Volume) bool {
	for _, curPlugin := range inTreePlugins {
		if curPlugin.CanSupportInline(vol) {
			return true
		}
	}
	return false
}


// RepairVolumeHandle generates a correct volume handle based on node ID information.
func (CSITranslator) RepairVolumeHandle(driverName, volumeHandle, nodeID string) (string, error) {
	if plugin, ok := inTreePlugins[driverName]; ok {
		return plugin.RepairVolumeHandle(volumeHandle, nodeID)
	}
	return "", fmt.Errorf("could not find In-Tree driver name for CSI plugin %v", driverName)
}


func New(

// New creates a new controller.
func New(
	ctx context.Context,
	name string,
	driver Driver,
	kubeClient kubernetes.Interface,
	informerFactory informers.SharedInformerFactory) Controller {
	logger := klog.LoggerWithName(klog.FromContext(ctx), "resource controller")
	rcInformer := informerFactory.Resource().V1alpha2().ResourceClasses()
	claimInformer := informerFactory.Resource().V1alpha2().ResourceClaims()
	schedulingCtxInformer := informerFactory.Resource().V1alpha2().PodSchedulingContexts()

	eventBroadcaster := record.NewBroadcaster()
	go func() {
		<-ctx.Done()
		eventBroadcaster.Shutdown()
	}()
	// TODO: use contextual logging in eventBroadcaster once it
	// supports it. There is a StartStructuredLogging API, but it
	// uses the global klog, which is worse than redirecting an unstructured
	// string into our logger, in particular during testing.
	eventBroadcaster.StartLogging(func(format string, args ...interface{}) {
		helper, logger := logger.WithCallStackHelper()
		helper()
		logger.V(2).Info(fmt.Sprintf(format, args...))
	})
	eventBroadcaster.StartRecordingToSink(&corev1types.EventSinkImpl{Interface: kubeClient.CoreV1().Events(v1.NamespaceAll)})
	eventRecorder := eventBroadcaster.NewRecorder(scheme.Scheme,
		v1.EventSource{Component: fmt.Sprintf("resource driver %s", name)})

	// The work queue contains either keys for claims or PodSchedulingContext objects.
	queue := workqueue.NewNamedRateLimitingQueue(
		workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("%s-queue", name))

	// The mutation cache acts as an additional layer for the informer
	// cache and after an update made by the controller returns a more
	// recent copy until the informer catches up.
	claimInformerCache := claimInformer.Informer().GetIndexer()
	claimCache := cache.NewIntegerResourceVersionMutationCache(claimInformerCache, claimInformerCache, 60*time.Second,
		false /* only cache updated claims that exist in the informer cache */)

	ctrl := &controller{
		ctx:                 ctx,
		logger:              logger,
		name:                name,
		finalizer:           name + "/deletion-protection",
		driver:              driver,
		kubeClient:          kubeClient,
		rcLister:            rcInformer.Lister(),
		rcSynced:            rcInformer.Informer().HasSynced,
		claimCache:          claimCache,
		claimSynced:         claimInformer.Informer().HasSynced,
		schedulingCtxLister: schedulingCtxInformer.Lister(),
		schedulingCtxSynced: schedulingCtxInformer.Informer().HasSynced,
		queue:               queue,
		eventRecorder:       eventRecorder,
	}

	loggerV6 := logger.V(6)
	if loggerV6.Enabled() {
		resourceClaimLogger := klog.LoggerWithValues(loggerV6, "type", "ResourceClaim")
		_, _ = claimInformer.Informer().AddEventHandler(resourceEventHandlerFuncs(&resourceClaimLogger, ctrl))
		schedulingCtxLogger := klog.LoggerWithValues(loggerV6, "type", "PodSchedulingContext")
		_, _ = schedulingCtxInformer.Informer().AddEventHandler(resourceEventHandlerFuncs(&schedulingCtxLogger, ctrl))
	} else {
		_, _ = claimInformer.Informer().AddEventHandler(resourceEventHandlerFuncs(nil, ctrl))
		_, _ = schedulingCtxInformer.Informer().AddEventHandler(resourceEventHandlerFuncs(nil, ctrl))
	}

	return ctrl
}


func resourceEventHandlerFuncs(logger *klog.Logger, ctrl *controller) cache.ResourceEventHandlerFuncs {
	return cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			ctrl.add(logger, obj)
		},
		UpdateFunc: func(oldObj, newObj interface{}) {
			ctrl.update(logger, oldObj, newObj)
		},
		DeleteFunc: ctrl.delete,
	}
}


func (ctrl *controller) add(logger *klog.Logger, obj interface{}) {
	if logger != nil {
		logger.Info("new object", "content", prettyPrint(obj))
	}
	ctrl.addNewOrUpdated("Adding new work item", obj)
}


func (ctrl *controller) update(logger *klog.Logger, oldObj, newObj interface{}) {
	if logger != nil {
		diff := cmp.Diff(oldObj, newObj)
		logger.Info("updated object", "content", prettyPrint(newObj), "diff", diff)
	}
	ctrl.addNewOrUpdated("Adding updated work item", newObj)
}


func (ctrl *controller) addNewOrUpdated(msg string, obj interface{}) {
	objKey, err := getKey(obj)
	if err != nil {
		ctrl.logger.Error(err, "Failed to get key", "obj", obj)
		return
	}
	ctrl.logger.V(5).Info(msg, "key", objKey)
	ctrl.queue.Add(objKey)
}


func (ctrl *controller) delete(obj interface{}) {
	objKey, err := getKey(obj)
	if err != nil {
		return
	}
	ctrl.logger.V(5).Info("Removing deleted work item", "key", objKey)
	ctrl.queue.Forget(objKey)
}


func getKey(obj interface{}) (string, error) {
	objKey, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
	if err != nil {
		return "", err
	}
	prefix := ""
	switch obj.(type) {
	case *resourcev1alpha2.ResourceClaim:
		prefix = claimKeyPrefix
	case *resourcev1alpha2.PodSchedulingContext:
		prefix = schedulingCtxKeyPrefix
	default:
		return "", fmt.Errorf("unexpected object: %T", obj)
	}

	return prefix + objKey, nil
}


// Run starts the controller.
func (ctrl *controller) Run(workers int) {
	defer ctrl.queue.ShutDown()

	ctrl.logger.Info("Starting", "driver", ctrl.name)
	defer ctrl.logger.Info("Shutting down", "driver", ctrl.name)

	stopCh := ctrl.ctx.Done()

	if !cache.WaitForCacheSync(stopCh, ctrl.rcSynced, ctrl.claimSynced, ctrl.schedulingCtxSynced) {
		ctrl.logger.Error(nil, "Cannot sync caches")
		return
	}

	for i := 0; i < workers; i++ {
		go wait.Until(ctrl.sync, 0, stopCh)
	}

	<-stopCh
}


func (ctrl *controller) sync()

// sync is the main worker.
func (ctrl *controller) sync() {
	key, quit := ctrl.queue.Get()
	if quit {
		return
	}
	defer ctrl.queue.Done(key)

	logger := klog.LoggerWithValues(ctrl.logger, "key", key)
	ctx := klog.NewContext(ctrl.ctx, logger)
	logger.V(4).Info("processing")
	obj, err := ctrl.syncKey(ctx, key.(string))
	switch err {
	case nil:
		logger.V(5).Info("completed")
		ctrl.queue.Forget(key)
	case errRequeue:
		logger.V(5).Info("requeue")
		ctrl.queue.AddRateLimited(key)
	case errPeriodic:
		logger.V(5).Info("recheck periodically")
		ctrl.queue.AddAfter(key, recheckDelay)
	default:
		logger.Error(err, "processing failed")
		if obj != nil {
			// TODO: We don't know here *what* failed. Determine based on error?
			ctrl.eventRecorder.Event(obj, v1.EventTypeWarning, "Failed", err.Error())
		}
		ctrl.queue.AddRateLimited(key)
	}
}


// syncKey looks up a ResourceClaim by its key and processes it.
func (ctrl *controller) syncKey(ctx context.Context, key string) (obj runtime.Object, finalErr error) {
	sep := strings.Index(key, ":")
	if sep < 0 {
		return nil, fmt.Errorf("unexpected key: %s", key)
	}
	prefix, object := key[0:sep+1], key[sep+1:]
	namespace, name, err := cache.SplitMetaNamespaceKey(object)
	if err != nil {
		return nil, err
	}

	switch prefix {
	case claimKeyPrefix:
		claim, err := ctrl.getCachedClaim(ctx, object)
		if claim == nil || err != nil {
			return nil, err
		}
		obj, finalErr = claim, ctrl.syncClaim(ctx, claim)
	case schedulingCtxKeyPrefix:
		schedulingCtx, err := ctrl.schedulingCtxLister.PodSchedulingContexts(namespace).Get(name)
		if err != nil {
			if k8serrors.IsNotFound(err) {
				klog.FromContext(ctx).V(5).Info("PodSchedulingContext was deleted, no need to process it")
				return nil, nil
			}
			return nil, err
		}
		obj, finalErr = schedulingCtx, ctrl.syncPodSchedulingContexts(ctx, schedulingCtx)
	}
	return
}


func (ctrl *controller) getCachedClaim(ctx context.Context, key string) (*resourcev1alpha2.ResourceClaim, error) {
	claimObj, exists, err := ctrl.claimCache.GetByKey(key)
	if !exists || k8serrors.IsNotFound(err) {
		klog.FromContext(ctx).V(5).Info("ResourceClaim not found, no need to process it")
		return nil, nil
	}
	if err != nil {
		return nil, err
	}
	claim, ok := claimObj.(*resourcev1alpha2.ResourceClaim)
	if !ok {
		return nil, fmt.Errorf("internal error: got %T instead of *resourcev1alpha2.ResourceClaim from claim cache", claimObj)
	}
	return claim, nil
}


// syncClaim determines which next action may be needed for a ResourceClaim
// and does it.
func (ctrl *controller) syncClaim(ctx context.Context, claim *resourcev1alpha2.ResourceClaim) error {
	var err error
	logger := klog.FromContext(ctx)

	if len(claim.Status.ReservedFor) > 0 {
		// In use. Nothing that we can do for it now.
		if loggerV6 := logger.V(6); loggerV6.Enabled() {
			loggerV6.Info("ResourceClaim in use", "reservedFor", claim.Status.ReservedFor)
		} else {
			logger.V(5).Info("ResourceClaim in use")
		}
		return nil
	}

	if claim.DeletionTimestamp != nil ||
		claim.Status.DeallocationRequested {
		// Ready for deallocation. We might have our finalizer set. The
		// finalizer is specific to the driver, therefore we know that
		// this claim is "ours" when the finalizer is set.
		hasFinalizer := ctrl.hasFinalizer(claim)
		logger.V(5).Info("ResourceClaim ready for deallocation", "deallocationRequested", claim.Status.DeallocationRequested, "deletionTimestamp", claim.DeletionTimestamp, "allocated", claim.Status.Allocation != nil, "hasFinalizer", hasFinalizer)
		if hasFinalizer {
			claim = claim.DeepCopy()
			if claim.Status.Allocation != nil {
				// Allocation was completed. Deallocate before proceeding.
				if err := ctrl.driver.Deallocate(ctx, claim); err != nil {
					return fmt.Errorf("deallocate: %v", err)
				}
				claim.Status.Allocation = nil
				claim.Status.DriverName = ""
				claim.Status.DeallocationRequested = false
				claim, err = ctrl.kubeClient.ResourceV1alpha2().ResourceClaims(claim.Namespace).UpdateStatus(ctx, claim, metav1.UpdateOptions{})
				if err != nil {
					return fmt.Errorf("remove allocation: %v", err)
				}
				ctrl.claimCache.Mutation(claim)
			} else {
				// Ensure that there is no on-going allocation.
				if err := ctrl.driver.Deallocate(ctx, claim); err != nil {
					return fmt.Errorf("stop allocation: %v", err)
				}
			}

			if claim.Status.DeallocationRequested {
				// Still need to remove it.
				claim.Status.DeallocationRequested = false
				claim, err = ctrl.kubeClient.ResourceV1alpha2().ResourceClaims(claim.Namespace).UpdateStatus(ctx, claim, metav1.UpdateOptions{})
				if err != nil {
					return fmt.Errorf("remove deallocation: %v", err)
				}
				ctrl.claimCache.Mutation(claim)
			}

			claim.Finalizers = ctrl.removeFinalizer(claim.Finalizers)
			claim, err = ctrl.kubeClient.ResourceV1alpha2().ResourceClaims(claim.Namespace).Update(ctx, claim, metav1.UpdateOptions{})
			if err != nil {
				return fmt.Errorf("remove finalizer: %v", err)
			}
			ctrl.claimCache.Mutation(claim)
		}

		// Nothing further to do. The apiserver should remove it shortly.
		return nil

	}

	if claim.Status.Allocation != nil {
		logger.V(5).Info("ResourceClaim is allocated")
		return nil
	}
	if claim.Spec.AllocationMode != resourcev1alpha2.AllocationModeImmediate {
		logger.V(5).Info("ResourceClaim waiting for first consumer")
		return nil
	}

	// We need the ResourceClass to determine whether we should allocate it.
	class, err := ctrl.rcLister.Get(claim.Spec.ResourceClassName)
	if err != nil {
		return err
	}
	if class.DriverName != ctrl.name {
		// Not ours *at the moment*. This can change, so requeue and
		// check again. We could trigger a faster check when the
		// ResourceClass changes, but that shouldn't occur much in
		// practice and thus isn't worth the effort.
		//
		// We use exponential backoff because it is unlikely that
		// the ResourceClass changes much.
		logger.V(5).Info("ResourceClaim is handled by other driver", "driver", class.DriverName)
		return errRequeue
	}

	// Check parameters.
	claimParameters, classParameters, err := ctrl.getParameters(ctx, claim, class)
	if err != nil {
		return err
	}

	return ctrl.allocateClaim(ctx, claim, claimParameters, class, classParameters, "", nil)
}


func (ctrl *controller) getParameters(ctx context.Context, claim *resourcev1alpha2.ResourceClaim, class *resourcev1alpha2.ResourceClass) (claimParameters, classParameters interface{}, err error) {
	classParameters, err = ctrl.driver.GetClassParameters(ctx, class)
	if err != nil {
		err = fmt.Errorf("class parameters %s: %v", class.ParametersRef, err)
		return
	}
	claimParameters, err = ctrl.driver.GetClaimParameters(ctx, claim, class, classParameters)
	if err != nil {
		err = fmt.Errorf("claim parameters %s: %v", claim.Spec.ParametersRef, err)
		return
	}
	return
}


func (ctrl *controller) allocateClaim(ctx context.Context,
	claim *resourcev1alpha2.ResourceClaim, claimParameters interface{},
	class *resourcev1alpha2.ResourceClass, classParameters interface{},
	selectedNode string,
	selectedUser *resourcev1alpha2.ResourceClaimConsumerReference) error {
	logger := klog.FromContext(ctx)

	if claim.Status.Allocation != nil {
		// This can happen when two PodSchedulingContext objects trigger
		// allocation attempts (first one wins) or when we see the
		// update of the PodSchedulingContext object.
		logger.V(5).Info("Claim already allocated, nothing to do")
		return nil
	}

	claim = claim.DeepCopy()
	if !ctrl.hasFinalizer(claim) {
		// Set finalizer before doing anything. We continue with the updated claim.
		logger.V(5).Info("Adding finalizer")
		claim.Finalizers = append(claim.Finalizers, ctrl.finalizer)
		var err error
		claim, err = ctrl.kubeClient.ResourceV1alpha2().ResourceClaims(claim.Namespace).Update(ctx, claim, metav1.UpdateOptions{})
		if err != nil {
			return fmt.Errorf("add finalizer: %v", err)
		}
		ctrl.claimCache.Mutation(claim)
	}

	logger.V(5).Info("Allocating")
	allocation, err := ctrl.driver.Allocate(ctx, claim, claimParameters, class, classParameters, selectedNode)
	if err != nil {
		return fmt.Errorf("allocate: %v", err)
	}
	claim.Status.Allocation = allocation
	claim.Status.DriverName = ctrl.name
	if selectedUser != nil {
		claim.Status.ReservedFor = append(claim.Status.ReservedFor, *selectedUser)
	}
	logger.V(6).Info("Updating claim after allocation", "claim", claim)
	claim, err = ctrl.kubeClient.ResourceV1alpha2().ResourceClaims(claim.Namespace).UpdateStatus(ctx, claim, metav1.UpdateOptions{})
	if err != nil {
		return fmt.Errorf("add allocation: %v", err)
	}
	ctrl.claimCache.Mutation(claim)
	return nil
}


func (ctrl *controller) checkPodClaim(ctx context.Context, pod *v1.Pod, podClaim v1.PodResourceClaim) (*ClaimAllocation, error) {
	claimName := resourceclaim.Name(pod, &podClaim)
	key := pod.Namespace + "/" + claimName
	claim, err := ctrl.getCachedClaim(ctx, key)
	if claim == nil || err != nil {
		return nil, err
	}
	if podClaim.Source.ResourceClaimTemplateName != nil {
		if err := resourceclaim.IsForPod(pod, claim); err != nil {
			return nil, err
		}
	}
	if claim.Spec.AllocationMode != resourcev1alpha2.AllocationModeWaitForFirstConsumer {
		// Nothing to do for it as part of pod scheduling.
		return nil, nil
	}
	class, err := ctrl.rcLister.Get(claim.Spec.ResourceClassName)
	if err != nil {
		return nil, err
	}
	if class.DriverName != ctrl.name {
		return nil, nil
	}
	// Check parameters.
	claimParameters, classParameters, err := ctrl.getParameters(ctx, claim, class)
	if err != nil {
		return nil, err
	}
	return &ClaimAllocation{
		PodClaimName:    podClaim.Name,
		Claim:           claim,
		Class:           class,
		ClaimParameters: claimParameters,
		ClassParameters: classParameters,
	}, nil
}


// syncPodSchedulingContext determines which next action may be needed for a PodSchedulingContext object
// and does it.
func (ctrl *controller) syncPodSchedulingContexts(ctx context.Context, schedulingCtx *resourcev1alpha2.PodSchedulingContext) error {
	logger := klog.FromContext(ctx)

	// Ignore deleted objects.
	if schedulingCtx.DeletionTimestamp != nil {
		logger.V(5).Info("PodSchedulingContext marked for deletion")
		return nil
	}

	if schedulingCtx.Spec.SelectedNode == "" &&
		len(schedulingCtx.Spec.PotentialNodes) == 0 {
		// Nothing to do? Shouldn't occur.
		logger.V(5).Info("Waiting for scheduler to set fields")
		return nil
	}

	// Check pod.
	// TODO (?): use an informer - only useful when many (most?) pods have claims
	// TODO (?): let the scheduler copy all claim names + UIDs into PodSchedulingContext - then we don't need the pod
	pod, err := ctrl.kubeClient.CoreV1().Pods(schedulingCtx.Namespace).Get(ctx, schedulingCtx.Name, metav1.GetOptions{})
	if err != nil {
		return err
	}
	if pod.DeletionTimestamp != nil {
		logger.V(5).Info("Pod marked for deletion")
		return nil
	}

	// Still the owner?
	if !metav1.IsControlledBy(schedulingCtx, pod) {
		// Must be obsolete object, do nothing for it.
		logger.V(5).Info("Pod not owner, PodSchedulingContext is obsolete")
		return nil
	}

	// Find all pending claims that are owned by us. We bail out if any of the pre-requisites
	// for pod scheduling (claims exist, classes exist, parameters exist) are not met.
	// The scheduler will do the same, except for checking parameters, so usually
	// everything should be ready once the PodSchedulingContext object exists.
	var claims claimAllocations
	for _, podClaim := range pod.Spec.ResourceClaims {
		delayed, err := ctrl.checkPodClaim(ctx, pod, podClaim)
		if err != nil {
			return fmt.Errorf("pod claim %s: %v", podClaim.Name, err)
		}
		if delayed == nil {
			// Nothing to do for it. This can change, so keep checking.
			continue
		}
		claims = append(claims, delayed)
	}
	if len(claims) == 0 {
		logger.V(5).Info("Found no pending pod claims")
		return errPeriodic
	}

	// Check current resource availability *before* triggering the
	// allocations. If we find that any of the claims cannot be allocated
	// for the selected node, we don't need to try for the others either
	// and shouldn't, because those allocations might have to be undone to
	// pick a better node. If we don't need to allocate now, then we'll
	// simply report back the gather information.
	if len(schedulingCtx.Spec.PotentialNodes) > 0 {
		if err := ctrl.driver.UnsuitableNodes(ctx, pod, claims, schedulingCtx.Spec.PotentialNodes); err != nil {
			return fmt.Errorf("checking potential nodes: %v", err)
		}
	}
	selectedNode := schedulingCtx.Spec.SelectedNode
	logger.V(5).Info("pending pod claims", "claims", claims, "selectedNode", selectedNode)
	if selectedNode != "" {
		unsuitable := false
		for _, delayed := range claims {
			if hasString(delayed.UnsuitableNodes, selectedNode) {
				unsuitable = true
				break
			}
		}

		if unsuitable {
			logger.V(2).Info("skipping allocation for unsuitable selected node", "node", selectedNode)
		} else {
			logger.V(2).Info("allocation for selected node", "node", selectedNode)
			selectedUser := &resourcev1alpha2.ResourceClaimConsumerReference{
				Resource: "pods",
				Name:     pod.Name,
				UID:      pod.UID,
			}
			for _, delayed := range claims {
				if err := ctrl.allocateClaim(ctx, delayed.Claim, delayed.ClaimParameters, delayed.Class, delayed.ClassParameters, selectedNode, selectedUser); err != nil {
					return fmt.Errorf("allocation of pod claim %s failed: %v", delayed.PodClaimName, err)
				}
			}
		}
	}

	// Now update unsuitable nodes. This is useful information for the scheduler even if
	// we managed to allocate because we might have to undo that.
	// TODO: replace with patching the array. We can do that without race conditions
	// because each driver is responsible for its own entries.
	modified := false
	schedulingCtx = schedulingCtx.DeepCopy()
	for _, delayed := range claims {
		i := findClaim(schedulingCtx.Status.ResourceClaims, delayed.PodClaimName)
		if i < 0 {
			// Add new entry.
			schedulingCtx.Status.ResourceClaims = append(schedulingCtx.Status.ResourceClaims,
				resourcev1alpha2.ResourceClaimSchedulingStatus{
					Name:            delayed.PodClaimName,
					UnsuitableNodes: delayed.UnsuitableNodes,
				})
			modified = true
		} else if stringsDiffer(schedulingCtx.Status.ResourceClaims[i].UnsuitableNodes, delayed.UnsuitableNodes) {
			// Update existing entry.
			schedulingCtx.Status.ResourceClaims[i].UnsuitableNodes = delayed.UnsuitableNodes
			modified = true
		}
	}
	if modified {
		logger.V(6).Info("Updating pod scheduling with modified unsuitable nodes", "podSchedulingCtx", schedulingCtx)
		if _, err := ctrl.kubeClient.ResourceV1alpha2().PodSchedulingContexts(schedulingCtx.Namespace).UpdateStatus(ctx, schedulingCtx, metav1.UpdateOptions{}); err != nil {
			return fmt.Errorf("update unsuitable node status: %v", err)
		}
	}

	// We must keep the object in our queue and keep updating the
	// UnsuitableNodes fields.
	return errPeriodic
}


// MarshalLog replaces the pointers with the actual structs because
// we care about the content, not the pointer values.
func (claims claimAllocations) MarshalLog() interface{} {
	content := make([]ClaimAllocation, 0, len(claims))
	for _, claim := range claims {
		content = append(content, *claim)
	}
	return content
}


// findClaim returns the index of the specified pod claim, -1 if not found.
func findClaim(claims []resourcev1alpha2.ResourceClaimSchedulingStatus, podClaimName string) int {
	for i := range claims {
		if claims[i].Name == podClaimName {
			return i
		}
	}
	return -1
}


// hasString checks for a string in a slice.
func hasString(strings []string, str string) bool {
	for _, s := range strings {
		if s == str {
			return true
		}
	}
	return false
}


func stringsDiffer(a, b []string) bool

// stringsDiffer does a strict comparison of two string arrays, order of entries matters.
func stringsDiffer(a, b []string) bool {
	if len(a) != len(b) {
		return true
	}
	for i := range a {
		if a[i] != b[i] {
			return true
		}
	}
	return false
}


// hasFinalizer checks if the claim has the finalizer of the driver.
func (ctrl *controller) hasFinalizer(claim *resourcev1alpha2.ResourceClaim) bool {
	for _, finalizer := range claim.Finalizers {
		if finalizer == ctrl.finalizer {
			return true
		}
	}
	return false
}


// removeFinalizer creates a new slice without the finalizer of the driver.
func (ctrl *controller) removeFinalizer(in []string) []string {
	out := make([]string, 0, len(in))
	for _, finalizer := range in {
		if finalizer != ctrl.finalizer {
			out = append(out, finalizer)
		}
	}
	if len(out) == 0 {
		return nil
	}
	return out
}


// prettyPrint formats arbitrary objects as JSON or, if that fails, with Sprintf.
func prettyPrint(obj interface{}) string {
	buffer, err := json.Marshal(obj)
	if err != nil {
		return fmt.Sprintf("%s", obj)
	}
	return string(buffer)
}


// DriverName defines the driver name for the dynamic resource allocation driver.
// Must be set.
func DriverName(driverName string) Option {
	return func(o *options) error {
		o.driverName = driverName
		return nil
	}
}


func Logger(logger klog.Logger) Option

// Logger overrides the default klog.Background logger.
func Logger(logger klog.Logger) Option {
	return func(o *options) error {
		o.logger = logger
		return nil
	}
}


func GRPCVerbosity(level int) Option

// GRPCVerbosity sets the verbosity for logging gRPC calls. Default is 4. A negative
// value disables logging.
func GRPCVerbosity(level int) Option {
	return func(o *options) error {
		o.grpcVerbosity = level
		return nil
	}
}


// RegistrarSocketPath sets the file path for a Unix domain socket.
// If RegistrarListener is not used, then Start will remove
// a file at that path, should one exist, and creates a socket
// itself. Otherwise it uses the provided listener and only
// removes the socket at the specified path during shutdown.
//
// At least one of these two options is required.
func RegistrarSocketPath(path string) Option {
	return func(o *options) error {
		o.pluginRegistrationEndpoint.path = path
		return nil
	}
}


// RegistrarListener sets an already created listener for the plugin
// registrarion API. Can be combined with RegistrarSocketPath.
//
// At least one of these two options is required.
func RegistrarListener(listener net.Listener) Option {
	return func(o *options) error {
		o.pluginRegistrationEndpoint.listener = listener
		return nil
	}
}


// PluginSocketPath sets the file path for a Unix domain socket.
// If PluginListener is not used, then Start will remove
// a file at that path, should one exist, and creates a socket
// itself. Otherwise it uses the provided listener and only
// removes the socket at the specified path during shutdown.
//
// At least one of these two options is required.
func PluginSocketPath(path string) Option {
	return func(o *options) error {
		o.draEndpoint.path = path
		return nil
	}
}


// PluginListener sets an already created listener for the dynamic resource
// allocation plugin API. Can be combined with PluginSocketPath.
//
// At least one of these two options is required.
func PluginListener(listener net.Listener) Option {
	return func(o *options) error {
		o.draEndpoint.listener = listener
		return nil
	}
}


// KubeletPluginSocketPath defines how kubelet will connect to the dynamic
// resource allocation plugin. This corresponds to PluginSocketPath, except
// that PluginSocketPath defines the path in the filesystem of the caller and
// KubeletPluginSocketPath in the filesystem of kubelet.
func KubeletPluginSocketPath(path string) Option {
	return func(o *options) error {
		o.draAddress = path
		return nil
	}
}


// GRPCInterceptor is called for each incoming gRPC method call.
func GRPCInterceptor(interceptor grpc.UnaryServerInterceptor) Option {
	return func(o *options) error {
		o.interceptor = interceptor
		return nil
	}
}


// Start sets up two gRPC servers (one for registration, one for the DRA node
// client).
func Start(nodeServer drapbv1.NodeServer, opts ...Option) (result DRAPlugin, finalErr error) {
	d := &draPlugin{}

	o := options{
		logger:        klog.Background(),
		grpcVerbosity: 4,
	}
	for _, option := range opts {
		if err := option(&o); err != nil {
			return nil, err
		}
	}

	if o.driverName == "" {
		return nil, errors.New("driver name must be set")
	}
	if o.draAddress == "" {
		return nil, errors.New("DRA address must be set")
	}
	var emptyEndpoint endpoint
	if o.draEndpoint == emptyEndpoint {
		return nil, errors.New("a Unix domain socket path and/or listener must be set for the kubelet plugin")
	}
	if o.pluginRegistrationEndpoint == emptyEndpoint {
		return nil, errors.New("a Unix domain socket path and/or listener must be set for the registrar")
	}

	// Run the node plugin gRPC server first to ensure that it is ready.
	plugin, err := startGRPCServer(klog.LoggerWithName(o.logger, "dra"), o.grpcVerbosity, o.interceptor, o.draEndpoint, func(grpcServer *grpc.Server) {
		drapbv1.RegisterNodeServer(grpcServer, nodeServer)
	})
	if err != nil {
		return nil, fmt.Errorf("start node client: %v", err)
	}
	d.plugin = plugin
	defer func() {
		// Clean up if we didn't finish succcessfully.
		if r := recover(); r != nil {
			plugin.stop()
			panic(r)
		}
		if finalErr != nil {
			plugin.stop()
		}
	}()

	// Now make it available to kubelet.
	registrar, err := startRegistrar(klog.LoggerWithName(o.logger, "registrar"), o.grpcVerbosity, o.interceptor, o.driverName, o.draAddress, o.pluginRegistrationEndpoint)
	if err != nil {
		return nil, fmt.Errorf("start registrar: %v", err)
	}
	d.registrar = registrar

	return d, nil
}


func (d *draPlugin) Stop()

func (d *draPlugin) Stop() {
	if d == nil {
		return
	}
	d.registrar.stop()
	d.plugin.stop()
}


func (d *draPlugin) RegistrationStatus() *registerapi.RegistrationStatus {
	if d.registrar == nil {
		return nil
	}
	return d.registrar.status
}


// startRegistrar returns a running instance.
func startRegistrar(logger klog.Logger, grpcVerbosity int, interceptor grpc.UnaryServerInterceptor, driverName string, endpoint string, pluginRegistrationEndpoint endpoint) (*nodeRegistrar, error) {
	n := &nodeRegistrar{
		logger: logger,
		registrationServer: registrationServer{
			driverName:        driverName,
			endpoint:          endpoint,
			supportedVersions: []string{"1.0.0"}, // TODO: is this correct?
		},
	}
	s, err := startGRPCServer(logger, grpcVerbosity, interceptor, pluginRegistrationEndpoint, func(grpcServer *grpc.Server) {
		registerapi.RegisterRegistrationServer(grpcServer, n)
	})
	if err != nil {
		return nil, fmt.Errorf("start gRPC server: %v", err)
	}
	n.server = s
	return n, nil
}


func (s *nodeRegistrar) stop()

// stop ensures that the registrar is not running anymore and cleans up all resources.
// It is idempotent and may be called with a nil pointer.
func (s *nodeRegistrar) stop() {
	if s == nil {
		return
	}
	s.server.stop()
}


// startGRPCServer sets up the GRPC server on a Unix domain socket and spawns a goroutine
// which handles requests for arbitrary services.
func startGRPCServer(logger klog.Logger, grpcVerbosity int, interceptor grpc.UnaryServerInterceptor, endpoint endpoint, services ...registerService) (*grpcServer, error) {
	s := &grpcServer{
		logger:        logger,
		endpoint:      endpoint,
		grpcVerbosity: grpcVerbosity,
	}

	listener := endpoint.listener
	if listener == nil {
		// Remove any (probably stale) existing socket.
		if err := os.Remove(endpoint.path); err != nil && !os.IsNotExist(err) {
			return nil, fmt.Errorf("remove Unix domain socket: %v", err)
		}

		// Now we can use the endpoint for listening.
		l, err := net.Listen("unix", endpoint.path)
		if err != nil {
			return nil, fmt.Errorf("listen on %q: %v", endpoint.path, err)
		}
		listener = l
	}

	// Run a gRPC server. It will close the listening socket when
	// shutting down, so we don't need to do that.
	var opts []grpc.ServerOption
	var interceptors []grpc.UnaryServerInterceptor
	if grpcVerbosity >= 0 {
		interceptors = append(interceptors, s.interceptor)
	}
	if interceptor != nil {
		interceptors = append(interceptors, interceptor)
	}
	if len(interceptors) >= 0 {
		opts = append(opts, grpc.ChainUnaryInterceptor(interceptors...))
	}
	s.server = grpc.NewServer(opts...)
	for _, service := range services {
		service(s.server)
	}
	s.wg.Add(1)
	go func() {
		defer s.wg.Done()
		err := s.server.Serve(listener)
		if err != nil {
			logger.Error(err, "GRPC server failed")
		} else {
			logger.V(3).Info("GRPC server terminated gracefully")
		}
	}()

	logger.Info("GRPC server started")
	return s, nil
}


// interceptor is called for each request. It creates a logger with a unique,
// sequentially increasing request ID and adds that logger to the context. It
// also logs request and response.
func (s *grpcServer) interceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
	requestID := atomic.AddInt64(&s.requestID, 1)
	logger := klog.LoggerWithValues(s.logger, "requestID", requestID)
	ctx = klog.NewContext(ctx, logger)
	logger.V(s.grpcVerbosity).Info("handling request", "request", req)
	defer func() {
		if r := recover(); r != nil {
			logger.Error(nil, "handling request panicked", "panic", r, "request", req)
			panic(r)
		}
	}()
	resp, err = handler(ctx, req)
	if err != nil {
		logger.Error(err, "handling request failed", "request", req)
	} else {
		logger.V(s.grpcVerbosity).Info("handling request succeeded", "response", resp)
	}
	return
}


func (s *grpcServer) stop()

// stop ensures that the server is not running anymore and cleans up all resources.
// It is idempotent and may be called with a nil pointer.
func (s *grpcServer) stop() {
	if s == nil {
		return
	}
	if s.server != nil {
		s.server.Stop()
	}
	s.wg.Wait()
	s.server = nil
	if s.endpoint.path != "" {
		if err := os.Remove(s.endpoint.path); err != nil && !os.IsNotExist(err) {
			s.logger.Error(err, "remove Unix socket")
		}
	}
	s.logger.V(3).Info("GRPC server stopped")
}


// GetInfo is the RPC invoked by plugin watcher.
func (e *registrationServer) GetInfo(ctx context.Context, req *registerapi.InfoRequest) (*registerapi.PluginInfo, error) {
	return &registerapi.PluginInfo{
		Type:              registerapi.DRAPlugin,
		Name:              e.driverName,
		Endpoint:          e.endpoint,
		SupportedVersions: e.supportedVersions,
	}, nil
}


// NotifyRegistrationStatus is the RPC invoked by plugin watcher.
func (e *registrationServer) NotifyRegistrationStatus(ctx context.Context, status *registerapi.RegistrationStatus) (*registerapi.RegistrationStatusResponse, error) {
	e.status = status
	if !status.PluginRegistered {
		return nil, fmt.Errorf("failed registration process: %+v", status.Error)
	}

	return &registerapi.RegistrationStatusResponse{}, nil
}


// New constructs a new leader election instance.
func New(clientset kubernetes.Interface, lockName string, runFunc func(ctx context.Context), opts ...Option) *leaderElection {
	l := &leaderElection{
		runFunc:            runFunc,
		lockName:           lockName,
		resourceLock:       resourcelock.LeasesResourceLock,
		leaseDuration:      defaultLeaseDuration,
		renewDeadline:      defaultRenewDeadline,
		retryPeriod:        defaultRetryPeriod,
		healthCheckTimeout: defaultHealthCheckTimeout,
		clientset:          clientset,
	}

	for _, opt := range opts {
		opt(l)
	}

	return l
}


func Identity(identity string) Option

func Identity(identity string) Option {
	return func(l *leaderElection) {
		l.identity = identity
	}
}


func Namespace(namespace string) Option

func Namespace(namespace string) Option {
	return func(l *leaderElection) {
		l.namespace = namespace
	}
}


func LeaseDuration(leaseDuration time.Duration) Option {
	return func(l *leaderElection) {
		l.leaseDuration = leaseDuration
	}
}


func RenewDeadline(renewDeadline time.Duration) Option {
	return func(l *leaderElection) {
		l.renewDeadline = renewDeadline
	}
}


func RetryPeriod(retryPeriod time.Duration) Option {
	return func(l *leaderElection) {
		l.retryPeriod = retryPeriod
	}
}


func HealthCheckTimeout(timeout time.Duration) Option {
	return func(l *leaderElection) {
		l.healthCheckTimeout = timeout
	}
}


func Context(ctx context.Context) Option {
	return func(l *leaderElection) {
		l.ctx = ctx
	}
}


// PrepareHealthCheck creates a health check for this leader election object
// with the given healthCheckTimeout and registers its HTTP handler to the given
// server at the path specified by the constant "healthCheckerAddress".
// healthCheckTimeout determines the max duration beyond lease expiration
// allowed before reporting unhealthy.
// The caller sidecar should document the handler address in appropriate flag
// descriptions.
func (l *leaderElection) PrepareHealthCheck(s Server) {
	l.healthCheck = leaderelection.NewLeaderHealthzAdaptor(l.healthCheckTimeout)
	s.Handle(HealthCheckerAddress, adaptCheckToHandler(l.healthCheck.Check))
}


func (l *leaderElection) Run() error

func (l *leaderElection) Run() error {
	if l.identity == "" {
		id, err := defaultLeaderElectionIdentity()
		if err != nil {
			return fmt.Errorf("error getting the default leader identity: %v", err)
		}

		l.identity = id
	}

	if l.namespace == "" {
		l.namespace = inClusterNamespace()
	}

	broadcaster := record.NewBroadcaster()
	broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: l.clientset.CoreV1().Events(l.namespace)})
	eventRecorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: fmt.Sprintf("%s/%s", l.lockName, l.identity)})

	rlConfig := resourcelock.ResourceLockConfig{
		Identity:      sanitizeName(l.identity),
		EventRecorder: eventRecorder,
	}

	lock, err := resourcelock.New(l.resourceLock, l.namespace, sanitizeName(l.lockName), l.clientset.CoreV1(), l.clientset.CoordinationV1(), rlConfig)
	if err != nil {
		return err
	}

	ctx := l.ctx
	if ctx == nil {
		ctx = context.Background()
	}
	leaderConfig := leaderelection.LeaderElectionConfig{
		Lock:          lock,
		LeaseDuration: l.leaseDuration,
		RenewDeadline: l.renewDeadline,
		RetryPeriod:   l.retryPeriod,
		Callbacks: leaderelection.LeaderCallbacks{
			OnStartedLeading: func(ctx context.Context) {
				klog.FromContext(ctx).Info("became leader, starting")
				l.runFunc(ctx)
			},
			OnStoppedLeading: func() {
				klog.FromContext(ctx).Error(nil, "stopped leading")
				klog.FlushAndExit(klog.ExitFlushTimeout, 1)
			},
			OnNewLeader: func(identity string) {
				klog.FromContext(ctx).Info("new leader detected", "idendity", identity)
			},
		},
		WatchDog: l.healthCheck,
	}

	leaderelection.RunOrDie(ctx, leaderConfig)
	return nil // should never reach here
}


func defaultLeaderElectionIdentity() (string, error) {
	return os.Hostname()
}


func sanitizeName(name string) string

// sanitizeName sanitizes the provided string so it can be consumed by leader election library
func sanitizeName(name string) string {
	re := regexp.MustCompile("[^a-zA-Z0-9-]")
	name = re.ReplaceAllString(name, "-")
	if name[len(name)-1] == '-' {
		// name must not end with '-'
		name = name + "X"
	}
	return name
}


func inClusterNamespace() string

// inClusterNamespace returns the namespace in which the pod is running in by checking
// the env var POD_NAMESPACE, then the file /var/run/secrets/kubernetes.io/serviceaccount/namespace.
// if neither returns a valid namespace, the "default" namespace is returned
func inClusterNamespace() string {
	if ns := os.Getenv("POD_NAMESPACE"); ns != "" {
		return ns
	}

	if data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
		if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
			return ns
		}
	}

	return "default"
}


// adaptCheckToHandler returns an http.HandlerFunc that serves the provided checks.
func adaptCheckToHandler(c func(r *http.Request) error) http.HandlerFunc {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		err := c(r)
		if err != nil {
			http.Error(w, fmt.Sprintf("internal server error: %v", err), http.StatusInternalServerError)
		} else {
			fmt.Fprint(w, "ok")
		}
	})
}


// Name returns the name of the ResourceClaim object that gets referenced by or
// created for the PodResourceClaim. The name is deterministic and therefore
// this function does not need any additional information and it will never
// fail.
//
// Either podClaim.ResourceClaimName or podClaim.Template must be non-nil, but not
// both. This is enforced by API validation.
//
// If podClaim.Template is not nil, the caller must check that the
// ResourceClaim is indeed the one that was created for the Pod by calling
// IsUsable.
func Name(pod *v1.Pod, podClaim *v1.PodResourceClaim) string {
	if podClaim.Source.ResourceClaimName != nil {
		return *podClaim.Source.ResourceClaimName
	}
	return pod.Name + "-" + podClaim.Name
}


// IsForPod checks that the ResourceClaim is the ephemeral volume that
// was created for the Pod. It returns an error that is informative
// enough to be returned by the caller without adding further details
// about the Pod or ResourceClaim.
func IsForPod(pod *v1.Pod, claim *resourcev1alpha2.ResourceClaim) error {
	// Checking the namespaces is just a precaution. The caller should
	// never pass in a ResourceClaim that isn't from the same namespace as the
	// Pod.
	if claim.Namespace != pod.Namespace || !metav1.IsControlledBy(claim, pod) {
		return fmt.Errorf("ResourceClaim %s/%s was not created for pod %s/%s (pod is not owner)", claim.Namespace, claim.Name, pod.Namespace, pod.Name)
	}
	return nil
}


// IsReservedForPod checks whether a claim lists the pod as one of the objects
// that the claim was reserved for.
func IsReservedForPod(pod *v1.Pod, claim *resourcev1alpha2.ResourceClaim) bool {
	for _, reserved := range claim.Status.ReservedFor {
		if reserved.UID == pod.UID {
			return true
		}
	}
	return false
}


// CanBeReserved checks whether the claim could be reserved for another object.
func CanBeReserved(claim *resourcev1alpha2.ResourceClaim) bool {
	return claim.Status.Allocation.Shareable ||
		len(claim.Status.ReservedFor) == 0
}


// IsVersionCheckMethod determines whether the supplied method is a version check against kms-plugin.
func IsVersionCheckMethod(method string) bool {
	return method == "/v1beta1.KeyManagementService/Version"
}


func (s *mockAESRemoteService) Encrypt(ctx context.Context, uid string, plaintext []byte) (*service.EncryptResponse, error) {
	out, err := s.transformer.TransformToStorage(ctx, plaintext, s.dataCtx)
	if err != nil {
		return nil, err
	}

	return &service.EncryptResponse{
		KeyID:      s.keyID,
		Ciphertext: out,
		Annotations: map[string][]byte{
			mockAnnotationKey: []byte("1"),
		},
	}, nil
}


func (s *mockAESRemoteService) Decrypt(ctx context.Context, uid string, req *service.DecryptRequest) ([]byte, error) {
	if len(req.Annotations) != 1 {
		return nil, errors.New("invalid annotations")
	}
	if v, ok := req.Annotations[mockAnnotationKey]; !ok || string(v) != "1" {
		return nil, errors.New("invalid version in annotations")
	}
	if req.KeyID != s.keyID {
		return nil, errors.New("invalid keyID")
	}
	from, _, err := s.transformer.TransformFromStorage(ctx, req.Ciphertext, s.dataCtx)
	if err != nil {
		return nil, err
	}
	return from, nil
}


func (s *mockAESRemoteService) Status(ctx context.Context) (*service.StatusResponse, error) {
	resp := &service.StatusResponse{
		Version: "v2beta1",
		Healthz: "ok",
		KeyID:   s.keyID,
	}
	return resp, nil
}


// NewMockAESService creates an instance of mockAESRemoteService.
func NewMockAESService(aesKey string, keyID string) (service.Service, error) {
	block, err := aes.NewCipher([]byte(aesKey))
	if err != nil {
		return nil, err
	}
	if len(keyID) == 0 {
		return nil, errors.New("invalid keyID")
	}
	return &mockAESRemoteService{
		transformer: aestransformer.NewGCMTransformer(block),
		keyID:       keyID,
		dataCtx:     value.DefaultContext([]byte{}),
	}, nil
}


func (s *mockLatencyRemoteService) Decrypt(ctx context.Context, uid string, req *service.DecryptRequest) ([]byte, error) {
	time.Sleep(s.latency)
	return s.delegate.Decrypt(ctx, uid, req)
}


func (s *mockLatencyRemoteService) Encrypt(ctx context.Context, uid string, data []byte) (*service.EncryptResponse, error) {
	time.Sleep(s.latency)
	return s.delegate.Encrypt(ctx, uid, data)
}


func (s *mockLatencyRemoteService) Status(ctx context.Context) (*service.StatusResponse, error) {
	// Passthrough here, not adding any delays for status as delays are usually negligible compare to encrypt and decrypt requests.
	return s.delegate.Status(ctx)
}


// NewMockLatencyService creates an instance of mockLatencyRemoteService.
func NewMockLatencyService(delegate service.Service, latency time.Duration) service.Service {
	return &mockLatencyRemoteService{
		delegate: delegate,
		latency:  latency,
	}
}


func (s *mockRateLimitRemoteService) Decrypt(ctx context.Context, uid string, req *service.DecryptRequest) ([]byte, error) {
	if !s.limiter.TryAccept() {
		return nil, status.New(codes.ResourceExhausted, "remote decrypt rate limit exceeded").Err()
	}
	return s.delegate.Decrypt(ctx, uid, req)
}


func (s *mockRateLimitRemoteService) Encrypt(ctx context.Context, uid string, data []byte) (*service.EncryptResponse, error) {
	if !s.limiter.TryAccept() {
		return nil, status.New(codes.ResourceExhausted, "remote encrypt rate limit exceeded").Err()
	}
	return s.delegate.Encrypt(ctx, uid, data)
}


func (s *mockRateLimitRemoteService) Status(ctx context.Context) (*service.StatusResponse, error) {
	// Passthrough here, not adding any rate limiting for status as rate limits are usually for encrypt and decrypt requests.
	return s.delegate.Status(ctx)
}


// NewMockRateLimitService creates an instance of mockRateLimitRemoteService.
func NewMockRateLimitService(delegate service.Service, qps float32, burst int) service.Service {
	return &mockRateLimitRemoteService{
		delegate: delegate,
		limiter:  flowcontrol.NewTokenBucketRateLimiter(qps, burst),
	}
}


func main()

func main() {
	flag.Parse()

	addr, err := util.ParseEndpoint(*listenAddr)
	if err != nil {
		klog.ErrorS(err, "failed to parse endpoint")
		os.Exit(1)
	}

	remoteKMSService, err := internal.NewMockAESService("somerandomstring", "aes-key-id")
	if err != nil {
		klog.ErrorS(err, "failed to create remote service")
		os.Exit(1)
	}

	ctx := withShutdownSignal(context.Background())
	grpcService := service.NewGRPCService(
		addr,
		*timeout,
		remoteKMSService,
	)

	klog.InfoS("starting server", "listenAddr", *listenAddr)
	go func() {
		if err := grpcService.ListenAndServe(); err != nil {
			klog.ErrorS(err, "failed to serve")
			os.Exit(1)
		}
	}()

	<-ctx.Done()
	klog.InfoS("shutting down server")
	grpcService.Shutdown()
}


// withShutdownSignal returns a copy of the parent context that will close if
// the process receives termination signals.
func withShutdownSignal(ctx context.Context) context.Context {
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT, os.Interrupt)

	nctx, cancel := context.WithCancel(ctx)

	go func() {
		<-signalChan
		klog.InfoS("received shutdown signal")
		cancel()
	}()
	return nctx
}


// NewGCMTransformer takes the given block cipher and performs encryption and decryption on the given
// data.
func NewGCMTransformer(block cipher.Block) value.Transformer {
	return &gcm{block: block}
}


func (t *gcm) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, bool, error) {
	aead, err := cipher.NewGCM(t.block)
	if err != nil {
		return nil, false, err
	}
	nonceSize := aead.NonceSize()
	if len(data) < nonceSize {
		return nil, false, fmt.Errorf("the stored data was shorter than the required size")
	}
	result, err := aead.Open(nil, data[:nonceSize], data[nonceSize:], dataCtx.AuthenticatedData())
	return result, false, err
}


func (t *gcm) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, error) {
	aead, err := cipher.NewGCM(t.block)
	if err != nil {
		return nil, err
	}
	nonceSize := aead.NonceSize()
	result := make([]byte, nonceSize+aead.Overhead()+len(data))
	n, err := rand.Read(result[:nonceSize])
	if err != nil {
		return nil, err
	}
	if n != nonceSize {
		return nil, fmt.Errorf("unable to read sufficient random bytes")
	}
	cipherText := aead.Seal(result[nonceSize:nonceSize], result[:nonceSize], data, dataCtx.AuthenticatedData())
	return result[:nonceSize+len(cipherText)], nil
}


func NewGRPCService(

// NewGRPCService creates an instance of GRPCService.
func NewGRPCService(
	address string,
	timeout time.Duration,

	kmsService Service,
) *GRPCService {
	klog.V(4).InfoS("KMS plugin configured", "address", address, "timeout", timeout)

	return &GRPCService{
		addr:       address,
		timeout:    timeout,
		kmsService: kmsService,
	}
}


// ListenAndServe accepts incoming connections on a Unix socket. It is a blocking method.
// Returns non-nil error unless Close or Shutdown is called.
func (s *GRPCService) ListenAndServe() error {
	ln, err := net.Listen("unix", s.addr)
	if err != nil {
		return err
	}
	defer ln.Close()

	gs := grpc.NewServer(
		grpc.ConnectionTimeout(s.timeout),
	)
	s.server = gs

	kmsapi.RegisterKeyManagementServiceServer(gs, s)

	klog.V(4).InfoS("kms plugin serving", "address", s.addr)
	return gs.Serve(ln)
}


func (s *GRPCService) Shutdown()

// Shutdown performs a graceful shutdown. Doesn't accept new connections and
// blocks until all pending RPCs are finished.
func (s *GRPCService) Shutdown() {
	klog.V(4).InfoS("kms plugin shutdown", "address", s.addr)
	if s.server != nil {
		s.server.GracefulStop()
	}
}


func (s *GRPCService) Close()

// Close stops the server by closing all connections immediately and cancels
// all active RPCs.
func (s *GRPCService) Close() {
	klog.V(4).InfoS("kms plugin close", "address", s.addr)
	if s.server != nil {
		s.server.Stop()
	}
}


// Status sends a status request to specified kms service.
func (s *GRPCService) Status(ctx context.Context, _ *kmsapi.StatusRequest) (*kmsapi.StatusResponse, error) {
	res, err := s.kmsService.Status(ctx)
	if err != nil {
		return nil, err
	}

	return &kmsapi.StatusResponse{
		Version: res.Version,
		Healthz: res.Healthz,
		KeyId:   res.KeyID,
	}, nil
}


// Decrypt sends a decryption request to specified kms service.
func (s *GRPCService) Decrypt(ctx context.Context, req *kmsapi.DecryptRequest) (*kmsapi.DecryptResponse, error) {
	klog.V(4).InfoS("decrypt request received", "id", req.Uid)

	plaintext, err := s.kmsService.Decrypt(ctx, req.Uid, &DecryptRequest{
		Ciphertext:  req.Ciphertext,
		KeyID:       req.KeyId,
		Annotations: req.Annotations,
	})
	if err != nil {
		return nil, err
	}

	return &kmsapi.DecryptResponse{
		Plaintext: plaintext,
	}, nil
}


// Encrypt sends an encryption request to specified kms service.
func (s *GRPCService) Encrypt(ctx context.Context, req *kmsapi.EncryptRequest) (*kmsapi.EncryptResponse, error) {
	klog.V(4).InfoS("encrypt request received", "id", req.Uid)

	encRes, err := s.kmsService.Encrypt(ctx, req.Uid, req.Plaintext)
	if err != nil {
		return nil, err
	}

	return &kmsapi.EncryptResponse{
		Ciphertext:  encRes.Ciphertext,
		KeyId:       encRes.KeyID,
		Annotations: encRes.Annotations,
	}, nil
}


// ParseEndpoint parses the endpoint to extract schema, host or path.
func ParseEndpoint(endpoint string) (string, error) {
	if len(endpoint) == 0 {
		return "", fmt.Errorf("remote KMS provider can't use empty string as endpoint")
	}

	u, err := url.Parse(endpoint)
	if err != nil {
		return "", fmt.Errorf("invalid endpoint %q for remote KMS provider, error: %v", endpoint, err)
	}

	if u.Scheme != unixProtocol {
		return "", fmt.Errorf("unsupported scheme %q for remote KMS provider", u.Scheme)
	}

	// Linux abstract namespace socket - no physical file required
	// Warning: Linux Abstract sockets have not concept of ACL (unlike traditional file based sockets).
	// However, Linux Abstract sockets are subject to Linux networking namespace, so will only be accessible to
	// containers within the same pod (unless host networking is used).
	if strings.HasPrefix(u.Path, "/@") {
		return strings.TrimPrefix(u.Path, "/"), nil
	}

	return u.Path, nil
}


// AuthenticatedData returns itself.
func (c DefaultContext) AuthenticatedData() []byte { return c }


func main()

func main() {
	stopCh := genericapiserver.SetupSignalHandler()
	options := server.NewDefaultOptions(os.Stdout, os.Stderr)
	cmd := server.NewCommandStartAggregator(options, stopCh)
	code := cli.Run(cmd)
	os.Exit(code)
}


// SortedByGroupAndVersion sorts APIServices into their different groups, and then sorts them based on their versions.
// For example, the first element of the first array contains the APIService with the highest version number, in the
// group with the highest priority; while the last element of the last array contains the APIService with the lowest
// version number, in the group with the lowest priority.
func SortedByGroupAndVersion(servers []*APIService) [][]*APIService {
	serversByGroupPriorityMinimum := ByGroupPriorityMinimum(servers)
	sort.Sort(serversByGroupPriorityMinimum)

	ret := [][]*APIService{}
	for _, curr := range serversByGroupPriorityMinimum {
		// check to see if we already have an entry for this group
		existingIndex := -1
		for j, groupInReturn := range ret {
			if groupInReturn[0].Spec.Group == curr.Spec.Group {
				existingIndex = j
				break
			}
		}

		if existingIndex >= 0 {
			ret[existingIndex] = append(ret[existingIndex], curr)
			sort.Sort(ByVersionPriority(ret[existingIndex]))
			continue
		}

		ret = append(ret, []*APIService{curr})
	}

	return ret
}


func (s ByGroupPriorityMinimum) Len() int      { return len(s) }


func (s ByGroupPriorityMinimum) Swap(i, j int) { s[i], s[j] = s[j], s[i] }


func (s ByGroupPriorityMinimum) Less(i, j int) bool {
	if s[i].Spec.GroupPriorityMinimum != s[j].Spec.GroupPriorityMinimum {
		return s[i].Spec.GroupPriorityMinimum > s[j].Spec.GroupPriorityMinimum
	}
	return s[i].Name < s[j].Name
}


func (s ByVersionPriority) Len() int      { return len(s) }


func (s ByVersionPriority) Swap(i, j int) { s[i], s[j] = s[j], s[i] }


func (s ByVersionPriority) Less(i, j int) bool {
	if s[i].Spec.VersionPriority != s[j].Spec.VersionPriority {
		return s[i].Spec.VersionPriority > s[j].Spec.VersionPriority
	}
	return version.CompareKubeAwareVersionStrings(s[i].Spec.Version, s[j].Spec.Version) > 0
}


// NewLocalAvailableAPIServiceCondition returns a condition for an available local APIService.
func NewLocalAvailableAPIServiceCondition() APIServiceCondition {
	return APIServiceCondition{
		Type:               Available,
		Status:             ConditionTrue,
		LastTransitionTime: metav1.Now(),
		Reason:             "Local",
		Message:            "Local APIServices are always available",
	}
}


// GetAPIServiceConditionByType gets an *APIServiceCondition by APIServiceConditionType if present
func GetAPIServiceConditionByType(apiService *APIService, conditionType APIServiceConditionType) *APIServiceCondition {
	for i := range apiService.Status.Conditions {
		if apiService.Status.Conditions[i].Type == conditionType {
			return &apiService.Status.Conditions[i]
		}
	}
	return nil
}


// SetAPIServiceCondition sets the status condition.  It either overwrites the existing one or
// creates a new one
func SetAPIServiceCondition(apiService *APIService, newCondition APIServiceCondition) {
	existingCondition := GetAPIServiceConditionByType(apiService, newCondition.Type)
	if existingCondition == nil {
		apiService.Status.Conditions = append(apiService.Status.Conditions, newCondition)
		return
	}

	if existingCondition.Status != newCondition.Status {
		existingCondition.Status = newCondition.Status
		existingCondition.LastTransitionTime = newCondition.LastTransitionTime
	}

	existingCondition.Reason = newCondition.Reason
	existingCondition.Message = newCondition.Message
}


// IsAPIServiceConditionTrue indicates if the condition is present and strictly true
func IsAPIServiceConditionTrue(apiService *APIService, conditionType APIServiceConditionType) bool {
	condition := GetAPIServiceConditionByType(apiService, conditionType)
	return condition != nil && condition.Status == ConditionTrue
}


func Install(scheme *runtime.Scheme)

// Install registers the API group and adds types to a scheme
func Install(scheme *runtime.Scheme) {
	utilruntime.Must(apiregistration.AddToScheme(scheme))
	utilruntime.Must(v1.AddToScheme(scheme))
	utilruntime.Must(v1beta1.AddToScheme(scheme))
	utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion, v1beta1.SchemeGroupVersion))
}


func Kind(kind string) schema.GroupKind

// Kind takes an unqualified kind and returns back a Group qualified GroupKind
func Kind(kind string) schema.GroupKind {
	return SchemeGroupVersion.WithKind(kind).GroupKind()
}


// Resource takes an unqualified resource and returns back a Group qualified GroupResource
func Resource(resource string) schema.GroupResource {
	return SchemeGroupVersion.WithResource(resource).GroupResource()
}


// Adds the list of known types to the given scheme.
func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&APIService{},
		&APIServiceList{},
	)
	return nil
}


func addDefaultingFuncs(scheme *runtime.Scheme) error {
	return RegisterDefaults(scheme)
}


// SetDefaults_ServiceReference sets defaults for AuditSync Webhook's ServiceReference
func SetDefaults_ServiceReference(obj *ServiceReference) {
	if obj.Port == nil {
		obj.Port = utilpointer.Int32Ptr(443)
	}
}


// SortedByGroupAndVersion sorts APIServices into their different groups, and then sorts them based on their versions.
// For example, the first element of the first array contains the APIService with the highest version number, in the
// group with the highest priority; while the last element of the last array contains the APIService with the lowest
// version number, in the group with the lowest priority.
func SortedByGroupAndVersion(servers []*v1.APIService) [][]*v1.APIService {
	serversByGroupPriorityMinimum := ByGroupPriorityMinimum(servers)
	sort.Sort(serversByGroupPriorityMinimum)

	ret := [][]*v1.APIService{}
	for _, curr := range serversByGroupPriorityMinimum {
		// check to see if we already have an entry for this group
		existingIndex := -1
		for j, groupInReturn := range ret {
			if groupInReturn[0].Spec.Group == curr.Spec.Group {
				existingIndex = j
				break
			}
		}

		if existingIndex >= 0 {
			ret[existingIndex] = append(ret[existingIndex], curr)
			sort.Sort(ByVersionPriority(ret[existingIndex]))
			continue
		}

		ret = append(ret, []*v1.APIService{curr})
	}

	return ret
}


func (s ByGroupPriorityMinimum) Len() int      { return len(s) }


func (s ByGroupPriorityMinimum) Swap(i, j int) { s[i], s[j] = s[j], s[i] }


func (s ByGroupPriorityMinimum) Less(i, j int) bool {
	if s[i].Spec.GroupPriorityMinimum != s[j].Spec.GroupPriorityMinimum {
		return s[i].Spec.GroupPriorityMinimum > s[j].Spec.GroupPriorityMinimum
	}
	return s[i].Name < s[j].Name
}


func (s ByVersionPriority) Len() int      { return len(s) }


func (s ByVersionPriority) Swap(i, j int) { s[i], s[j] = s[j], s[i] }


func (s ByVersionPriority) Less(i, j int) bool {
	if s[i].Spec.VersionPriority != s[j].Spec.VersionPriority {
		return s[i].Spec.VersionPriority > s[j].Spec.VersionPriority
	}
	return version.CompareKubeAwareVersionStrings(s[i].Spec.Version, s[j].Spec.Version) > 0
}


// APIServiceNameToGroupVersion returns the GroupVersion for a given apiServiceName.  The name
// must be valid, but any object you get back from an informer will be valid.
func APIServiceNameToGroupVersion(apiServiceName string) schema.GroupVersion {
	tokens := strings.SplitN(apiServiceName, ".", 2)
	return schema.GroupVersion{Group: tokens[1], Version: tokens[0]}
}


// NewLocalAvailableAPIServiceCondition returns a condition for an available local APIService.
func NewLocalAvailableAPIServiceCondition() v1.APIServiceCondition {
	return v1.APIServiceCondition{
		Type:               v1.Available,
		Status:             v1.ConditionTrue,
		LastTransitionTime: metav1.Now(),
		Reason:             "Local",
		Message:            "Local APIServices are always available",
	}
}


// SetAPIServiceCondition sets the status condition.  It either overwrites the existing one or
// creates a new one
func SetAPIServiceCondition(apiService *v1.APIService, newCondition v1.APIServiceCondition) {
	existingCondition := GetAPIServiceConditionByType(apiService, newCondition.Type)
	if existingCondition == nil {
		apiService.Status.Conditions = append(apiService.Status.Conditions, newCondition)
		return
	}

	if existingCondition.Status != newCondition.Status {
		existingCondition.Status = newCondition.Status
		existingCondition.LastTransitionTime = newCondition.LastTransitionTime
	}

	existingCondition.Reason = newCondition.Reason
	existingCondition.Message = newCondition.Message
}


// IsAPIServiceConditionTrue indicates if the condition is present and strictly true
func IsAPIServiceConditionTrue(apiService *v1.APIService, conditionType v1.APIServiceConditionType) bool {
	condition := GetAPIServiceConditionByType(apiService, conditionType)
	return condition != nil && condition.Status == v1.ConditionTrue
}


// GetAPIServiceConditionByType gets an *APIServiceCondition by APIServiceConditionType if present
func GetAPIServiceConditionByType(apiService *v1.APIService, conditionType v1.APIServiceConditionType) *v1.APIServiceCondition {
	for i := range apiService.Status.Conditions {
		if apiService.Status.Conditions[i].Type == conditionType {
			return &apiService.Status.Conditions[i]
		}
	}
	return nil
}


func init()

func init() {
	// We only register manually written functions here. The registration of the
	// generated functions takes place in the generated files. The separation
	// makes the code compile even when the generated files are missing.
	localSchemeBuilder.Register(addKnownTypes, addDefaultingFuncs)
}


// Adds the list of known types to the given scheme.
func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&APIService{},
		&APIServiceList{},
	)
	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
	return nil
}


// Resource takes an unqualified resource and returns back a Group qualified GroupResource
func Resource(resource string) schema.GroupResource {
	return SchemeGroupVersion.WithResource(resource).GroupResource()
}


// ValidateAPIService validates that the APIService is correctly defined.
func ValidateAPIService(apiService *apiregistration.APIService) field.ErrorList {
	requiredName := apiService.Spec.Version + "." + apiService.Spec.Group

	allErrs := validation.ValidateObjectMeta(&apiService.ObjectMeta, false,
		func(name string, prefix bool) []string {
			if minimalFailures := path.IsValidPathSegmentName(name); len(minimalFailures) > 0 {
				return minimalFailures
			}
			// the name *must* be version.group
			if name != requiredName {
				return []string{fmt.Sprintf("must be `spec.version+\".\"+spec.group`: %q", requiredName)}
			}

			return []string{}
		},
		field.NewPath("metadata"))

	// in this case we allow empty group
	if len(apiService.Spec.Group) == 0 && apiService.Spec.Version != "v1" {
		allErrs = append(allErrs, field.Required(field.NewPath("spec", "group"), "only v1 may have an empty group and it better be legacy kube"))
	}
	if len(apiService.Spec.Group) > 0 {
		for _, errString := range utilvalidation.IsDNS1123Subdomain(apiService.Spec.Group) {
			allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "group"), apiService.Spec.Group, errString))
		}
	}

	for _, errString := range utilvalidation.IsDNS1035Label(apiService.Spec.Version) {
		allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "version"), apiService.Spec.Version, errString))
	}

	if apiService.Spec.GroupPriorityMinimum <= 0 || apiService.Spec.GroupPriorityMinimum > 20000 {
		allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "groupPriorityMinimum"), apiService.Spec.GroupPriorityMinimum, "must be positive and less than 20000"))
	}
	if apiService.Spec.VersionPriority <= 0 || apiService.Spec.VersionPriority > 1000 {
		allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "versionPriority"), apiService.Spec.VersionPriority, "must be positive and less than 1000"))
	}

	if apiService.Spec.Service == nil {
		if len(apiService.Spec.CABundle) != 0 {
			allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "caBundle"), fmt.Sprintf("%d bytes", len(apiService.Spec.CABundle)), "local APIServices may not have a caBundle"))
		}
		if apiService.Spec.InsecureSkipTLSVerify {
			allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "insecureSkipTLSVerify"), apiService.Spec.InsecureSkipTLSVerify, "local APIServices may not have insecureSkipTLSVerify"))
		}
		return allErrs
	}

	if len(apiService.Spec.Service.Namespace) == 0 {
		allErrs = append(allErrs, field.Required(field.NewPath("spec", "service", "namespace"), ""))
	}
	if len(apiService.Spec.Service.Name) == 0 {
		allErrs = append(allErrs, field.Required(field.NewPath("spec", "service", "name"), ""))
	}
	if errs := utilvalidation.IsValidPortNum(int(apiService.Spec.Service.Port)); errs != nil {
		allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "service", "port"), apiService.Spec.Service.Port, "port is not valid: "+strings.Join(errs, ", ")))
	}
	if apiService.Spec.InsecureSkipTLSVerify && len(apiService.Spec.CABundle) > 0 {
		allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "insecureSkipTLSVerify"), apiService.Spec.InsecureSkipTLSVerify, "may not be true if caBundle is present"))
	}

	return allErrs
}


// ValidateAPIServiceUpdate validates an update of APIService.
func ValidateAPIServiceUpdate(newAPIService *apiregistration.APIService, oldAPIService *apiregistration.APIService) field.ErrorList {
	allErrs := validation.ValidateObjectMetaUpdate(&newAPIService.ObjectMeta, &oldAPIService.ObjectMeta, field.NewPath("metadata"))
	allErrs = append(allErrs, ValidateAPIService(newAPIService)...)

	return allErrs
}


// ValidateAPIServiceStatus validates that the APIService status is one of 'True', 'False' or 'Unknown'.
func ValidateAPIServiceStatus(status *apiregistration.APIServiceStatus, fldPath *field.Path) field.ErrorList {
	allErrs := field.ErrorList{}

	for i, condition := range status.Conditions {
		if condition.Status != apiregistration.ConditionTrue &&
			condition.Status != apiregistration.ConditionFalse &&
			condition.Status != apiregistration.ConditionUnknown {
			allErrs = append(allErrs, field.NotSupported(fldPath.Child("conditions").Index(i).Child("status"), condition.Status, []string{
				string(apiregistration.ConditionTrue), string(apiregistration.ConditionFalse), string(apiregistration.ConditionUnknown)}))
		}
	}

	return allErrs
}


// ValidateAPIServiceStatusUpdate validates an update of the status field of APIService.
func ValidateAPIServiceStatusUpdate(newAPIService *apiregistration.APIService, oldAPIService *apiregistration.APIService) field.ErrorList {
	allErrs := validation.ValidateObjectMetaUpdate(&newAPIService.ObjectMeta, &oldAPIService.ObjectMeta, field.NewPath("metadata"))
	allErrs = append(allErrs, ValidateAPIServiceStatus(&newAPIService.Status, field.NewPath("status"))...)
	return allErrs
}


func init() {
	// we need to add the options (like ListOptions) to empty v1
	metav1.AddToGroupVersion(aggregatorscheme.Scheme, schema.GroupVersion{Group: "", Version: "v1"})

	unversioned := schema.GroupVersion{Group: "", Version: "v1"}
	aggregatorscheme.Scheme.AddUnversionedTypes(unversioned,
		&metav1.Status{},
		&metav1.APIVersions{},
		&metav1.APIGroupList{},
		&metav1.APIGroup{},
		&metav1.APIResourceList{},
	)
}


// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
func (cfg *Config) Complete() CompletedConfig {
	c := completedConfig{
		cfg.GenericConfig.Complete(),
		&cfg.ExtraConfig,
	}

	// the kube aggregator wires its own discovery mechanism
	// TODO eventually collapse this by extracting all of the discovery out
	c.GenericConfig.EnableDiscovery = false
	version := version.Get()
	c.GenericConfig.Version = &version

	return CompletedConfig{&c}
}


// NewWithDelegate returns a new instance of APIAggregator from the given config.
func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.DelegationTarget) (*APIAggregator, error) {
	genericServer, err := c.GenericConfig.New("kube-aggregator", delegationTarget)
	if err != nil {
		return nil, err
	}

	apiregistrationClient, err := clientset.NewForConfig(c.GenericConfig.LoopbackClientConfig)
	if err != nil {
		return nil, err
	}
	informerFactory := informers.NewSharedInformerFactory(
		apiregistrationClient,
		5*time.Minute, // this is effectively used as a refresh interval right now.  Might want to do something nicer later on.
	)

	// apiServiceRegistrationControllerInitiated is closed when APIServiceRegistrationController has finished "installing" all known APIServices.
	// At this point we know that the proxy handler knows about APIServices and can handle client requests.
	// Before it might have resulted in a 404 response which could have serious consequences for some controllers like  GC and NS
	//
	// Note that the APIServiceRegistrationController waits for APIServiceInformer to synced before doing its work.
	apiServiceRegistrationControllerInitiated := make(chan struct{})
	if err := genericServer.RegisterMuxAndDiscoveryCompleteSignal("APIServiceRegistrationControllerInitiated", apiServiceRegistrationControllerInitiated); err != nil {
		return nil, err
	}

	var proxyTransportDial *transport.DialHolder
	if c.GenericConfig.EgressSelector != nil {
		egressDialer, err := c.GenericConfig.EgressSelector.Lookup(egressselector.Cluster.AsNetworkContext())
		if err != nil {
			return nil, err
		}
		if egressDialer != nil {
			proxyTransportDial = &transport.DialHolder{Dial: egressDialer}
		}
	} else if c.ExtraConfig.ProxyTransport != nil && c.ExtraConfig.ProxyTransport.DialContext != nil {
		proxyTransportDial = &transport.DialHolder{Dial: c.ExtraConfig.ProxyTransport.DialContext}
	}

	s := &APIAggregator{
		GenericAPIServer:           genericServer,
		delegateHandler:            delegationTarget.UnprotectedHandler(),
		proxyTransportDial:         proxyTransportDial,
		proxyHandlers:              map[string]*proxyHandler{},
		handledGroups:              sets.String{},
		lister:                     informerFactory.Apiregistration().V1().APIServices().Lister(),
		APIRegistrationInformers:   informerFactory,
		serviceResolver:            c.ExtraConfig.ServiceResolver,
		openAPIConfig:              c.GenericConfig.OpenAPIConfig,
		openAPIV3Config:            c.GenericConfig.OpenAPIV3Config,
		proxyCurrentCertKeyContent: func() (bytes []byte, bytes2 []byte) { return nil, nil },
		rejectForwardingRedirects:  c.ExtraConfig.RejectForwardingRedirects,
	}

	// used later  to filter the served resource by those that have expired.
	resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(*c.GenericConfig.Version)
	if err != nil {
		return nil, err
	}

	apiGroupInfo := apiservicerest.NewRESTStorage(c.GenericConfig.MergedResourceConfig, c.GenericConfig.RESTOptionsGetter, resourceExpirationEvaluator.ShouldServeForVersion(1, 22))
	if err := s.GenericAPIServer.InstallAPIGroup(&apiGroupInfo); err != nil {
		return nil, err
	}

	enabledVersions := sets.NewString()
	for v := range apiGroupInfo.VersionedResourcesStorageMap {
		enabledVersions.Insert(v)
	}
	if !enabledVersions.Has(v1.SchemeGroupVersion.Version) {
		return nil, fmt.Errorf("API group/version %s must be enabled", v1.SchemeGroupVersion.String())
	}

	apisHandler := &apisHandler{
		codecs:         aggregatorscheme.Codecs,
		lister:         s.lister,
		discoveryGroup: discoveryGroup(enabledVersions),
	}

	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
		apisHandlerWithAggregationSupport := aggregated.WrapAggregatedDiscoveryToHandler(apisHandler, s.GenericAPIServer.AggregatedDiscoveryGroupManager)
		s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/apis", apisHandlerWithAggregationSupport)
	} else {
		s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/apis", apisHandler)
	}
	s.GenericAPIServer.Handler.NonGoRestfulMux.UnlistedHandle("/apis/", apisHandler)

	apiserviceRegistrationController := NewAPIServiceRegistrationController(informerFactory.Apiregistration().V1().APIServices(), s)
	if len(c.ExtraConfig.ProxyClientCertFile) > 0 && len(c.ExtraConfig.ProxyClientKeyFile) > 0 {
		aggregatorProxyCerts, err := dynamiccertificates.NewDynamicServingContentFromFiles("aggregator-proxy-cert", c.ExtraConfig.ProxyClientCertFile, c.ExtraConfig.ProxyClientKeyFile)
		if err != nil {
			return nil, err
		}
		// We are passing the context to ProxyCerts.RunOnce as it needs to implement RunOnce(ctx) however the
		// context is not used at all. So passing a empty context shouldn't be a problem
		ctx := context.TODO()
		if err := aggregatorProxyCerts.RunOnce(ctx); err != nil {
			return nil, err
		}
		aggregatorProxyCerts.AddListener(apiserviceRegistrationController)
		s.proxyCurrentCertKeyContent = aggregatorProxyCerts.CurrentCertKeyContent

		s.GenericAPIServer.AddPostStartHookOrDie("aggregator-reload-proxy-client-cert", func(postStartHookContext genericapiserver.PostStartHookContext) error {
			// generate a context  from stopCh. This is to avoid modifying files which are relying on apiserver
			// TODO: See if we can pass ctx to the current method
			ctx, cancel := context.WithCancel(context.Background())
			go func() {
				select {
				case <-postStartHookContext.StopCh:
					cancel() // stopCh closed, so cancel our context
				case <-ctx.Done():
				}
			}()
			go aggregatorProxyCerts.Run(ctx, 1)
			return nil
		})
	}

	availableController, err := statuscontrollers.NewAvailableConditionController(
		informerFactory.Apiregistration().V1().APIServices(),
		c.GenericConfig.SharedInformerFactory.Core().V1().Services(),
		c.GenericConfig.SharedInformerFactory.Core().V1().Endpoints(),
		apiregistrationClient.ApiregistrationV1(),
		proxyTransportDial,
		(func() ([]byte, []byte))(s.proxyCurrentCertKeyContent),
		s.serviceResolver,
	)
	if err != nil {
		return nil, err
	}

	s.GenericAPIServer.AddPostStartHookOrDie("start-kube-aggregator-informers", func(context genericapiserver.PostStartHookContext) error {
		informerFactory.Start(context.StopCh)
		c.GenericConfig.SharedInformerFactory.Start(context.StopCh)
		return nil
	})
	s.GenericAPIServer.AddPostStartHookOrDie("apiservice-registration-controller", func(context genericapiserver.PostStartHookContext) error {
		go apiserviceRegistrationController.Run(context.StopCh, apiServiceRegistrationControllerInitiated)
		select {
		case <-context.StopCh:
		case <-apiServiceRegistrationControllerInitiated:
		}

		return nil
	})
	s.GenericAPIServer.AddPostStartHookOrDie("apiservice-status-available-controller", func(context genericapiserver.PostStartHookContext) error {
		// if we end up blocking for long periods of time, we may need to increase workers.
		go availableController.Run(5, context.StopCh)
		return nil
	})

	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StorageVersionAPI) &&
		utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) {
		// Spawn a goroutine in aggregator apiserver to update storage version for
		// all built-in resources
		s.GenericAPIServer.AddPostStartHookOrDie(StorageVersionPostStartHookName, func(hookContext genericapiserver.PostStartHookContext) error {
			// Wait for apiserver-identity to exist first before updating storage
			// versions, to avoid storage version GC accidentally garbage-collecting
			// storage versions.
			kubeClient, err := kubernetes.NewForConfig(hookContext.LoopbackClientConfig)
			if err != nil {
				return err
			}
			if err := wait.PollImmediateUntil(100*time.Millisecond, func() (bool, error) {
				_, err := kubeClient.CoordinationV1().Leases(metav1.NamespaceSystem).Get(
					context.TODO(), s.GenericAPIServer.APIServerID, metav1.GetOptions{})
				if apierrors.IsNotFound(err) {
					return false, nil
				}
				if err != nil {
					return false, err
				}
				return true, nil
			}, hookContext.StopCh); err != nil {
				return fmt.Errorf("failed to wait for apiserver-identity lease %s to be created: %v",
					s.GenericAPIServer.APIServerID, err)
			}
			// Technically an apiserver only needs to update storage version once during bootstrap.
			// Reconcile StorageVersion objects every 10 minutes will help in the case that the
			// StorageVersion objects get accidentally modified/deleted by a different agent. In that
			// case, the reconciliation ensures future storage migration still works. If nothing gets
			// changed, the reconciliation update is a noop and gets short-circuited by the apiserver,
			// therefore won't change the resource version and trigger storage migration.
			go wait.PollImmediateUntil(10*time.Minute, func() (bool, error) {
				// All apiservers (aggregator-apiserver, kube-apiserver, apiextensions-apiserver)
				// share the same generic apiserver config. The same StorageVersion manager is used
				// to register all built-in resources when the generic apiservers install APIs.
				s.GenericAPIServer.StorageVersionManager.UpdateStorageVersions(hookContext.LoopbackClientConfig, s.GenericAPIServer.APIServerID)
				return false, nil
			}, hookContext.StopCh)
			// Once the storage version updater finishes the first round of update,
			// the PostStartHook will return to unblock /healthz. The handler chain
			// won't block write requests anymore. Check every second since it's not
			// expensive.
			wait.PollImmediateUntil(1*time.Second, func() (bool, error) {
				return s.GenericAPIServer.StorageVersionManager.Completed(), nil
			}, hookContext.StopCh)
			return nil
		})
	}

	return s, nil
}


// PrepareRun prepares the aggregator to run, by setting up the OpenAPI spec &
// aggregated discovery document and calling the generic PrepareRun.
func (s *APIAggregator) PrepareRun() (preparedAPIAggregator, error) {
	// add post start hook before generic PrepareRun in order to be before /healthz installation
	if s.openAPIConfig != nil {
		s.GenericAPIServer.AddPostStartHookOrDie("apiservice-openapi-controller", func(context genericapiserver.PostStartHookContext) error {
			go s.openAPIAggregationController.Run(context.StopCh)
			return nil
		})
	}

	if s.openAPIV3Config != nil && utilfeature.DefaultFeatureGate.Enabled(genericfeatures.OpenAPIV3) {
		s.GenericAPIServer.AddPostStartHookOrDie("apiservice-openapiv3-controller", func(context genericapiserver.PostStartHookContext) error {
			go s.openAPIV3AggregationController.Run(context.StopCh)
			return nil
		})
	}

	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
		s.discoveryAggregationController = NewDiscoveryManager(
			// Use aggregator as the source name to avoid overwriting native/CRD
			// groups
			s.GenericAPIServer.AggregatedDiscoveryGroupManager.WithSource(aggregated.AggregatorSource),
		)

		// Setup discovery endpoint
		s.GenericAPIServer.AddPostStartHookOrDie("apiservice-discovery-controller", func(context genericapiserver.PostStartHookContext) error {
			// Run discovery manager's worker to watch for new/removed/updated
			// APIServices to the discovery document can be updated at runtime
			go s.discoveryAggregationController.Run(context.StopCh)
			return nil
		})
	}

	prepared := s.GenericAPIServer.PrepareRun()

	// delay OpenAPI setup until the delegate had a chance to setup their OpenAPI handlers
	if s.openAPIConfig != nil {
		specDownloader := openapiaggregator.NewDownloader()
		openAPIAggregator, err := openapiaggregator.BuildAndRegisterAggregator(
			&specDownloader,
			s.GenericAPIServer.NextDelegate(),
			s.GenericAPIServer.Handler.GoRestfulContainer.RegisteredWebServices(),
			s.openAPIConfig,
			s.GenericAPIServer.Handler.NonGoRestfulMux)
		if err != nil {
			return preparedAPIAggregator{}, err
		}
		s.openAPIAggregationController = openapicontroller.NewAggregationController(&specDownloader, openAPIAggregator)
	}

	if s.openAPIV3Config != nil && utilfeature.DefaultFeatureGate.Enabled(genericfeatures.OpenAPIV3) {
		specDownloaderV3 := openapiv3aggregator.NewDownloader()
		openAPIV3Aggregator, err := openapiv3aggregator.BuildAndRegisterAggregator(
			specDownloaderV3,
			s.GenericAPIServer.NextDelegate(),
			s.GenericAPIServer.Handler.NonGoRestfulMux)
		if err != nil {
			return preparedAPIAggregator{}, err
		}
		s.openAPIV3AggregationController = openapiv3controller.NewAggregationController(openAPIV3Aggregator)
	}

	return preparedAPIAggregator{APIAggregator: s, runnable: prepared}, nil
}


func (s preparedAPIAggregator) Run(stopCh <-chan struct{}) error {
	return s.runnable.Run(stopCh)
}


// AddAPIService adds an API service.  It is not thread-safe, so only call it on one thread at a time please.
// It's a slow moving API, so its ok to run the controller on a single thread
func (s *APIAggregator) AddAPIService(apiService *v1.APIService) error {
	// if the proxyHandler already exists, it needs to be updated. The aggregation bits do not
	// since they are wired against listers because they require multiple resources to respond
	if proxyHandler, exists := s.proxyHandlers[apiService.Name]; exists {
		proxyHandler.updateAPIService(apiService)
		if s.openAPIAggregationController != nil {
			s.openAPIAggregationController.UpdateAPIService(proxyHandler, apiService)
		}
		if s.openAPIV3AggregationController != nil {
			s.openAPIV3AggregationController.UpdateAPIService(proxyHandler, apiService)
		}
		// Forward calls to discovery manager to update discovery document
		if s.discoveryAggregationController != nil {
			handlerCopy := *proxyHandler
			handlerCopy.setServiceAvailable()
			s.discoveryAggregationController.AddAPIService(apiService, &handlerCopy)
		}
		return nil
	}

	proxyPath := "/apis/" + apiService.Spec.Group + "/" + apiService.Spec.Version
	// v1. is a special case for the legacy API.  It proxies to a wider set of endpoints.
	if apiService.Name == legacyAPIServiceName {
		proxyPath = "/api"
	}

	// register the proxy handler
	proxyHandler := &proxyHandler{
		localDelegate:              s.delegateHandler,
		proxyCurrentCertKeyContent: s.proxyCurrentCertKeyContent,
		proxyTransportDial:         s.proxyTransportDial,
		serviceResolver:            s.serviceResolver,
		rejectForwardingRedirects:  s.rejectForwardingRedirects,
	}
	proxyHandler.updateAPIService(apiService)
	if s.openAPIAggregationController != nil {
		s.openAPIAggregationController.AddAPIService(proxyHandler, apiService)
	}
	if s.openAPIV3AggregationController != nil {
		s.openAPIV3AggregationController.AddAPIService(proxyHandler, apiService)
	}
	if s.discoveryAggregationController != nil {
		s.discoveryAggregationController.AddAPIService(apiService, proxyHandler)
	}

	s.proxyHandlers[apiService.Name] = proxyHandler
	s.GenericAPIServer.Handler.NonGoRestfulMux.Handle(proxyPath, proxyHandler)
	s.GenericAPIServer.Handler.NonGoRestfulMux.UnlistedHandlePrefix(proxyPath+"/", proxyHandler)

	// if we're dealing with the legacy group, we're done here
	if apiService.Name == legacyAPIServiceName {
		return nil
	}

	// if we've already registered the path with the handler, we don't want to do it again.
	if s.handledGroups.Has(apiService.Spec.Group) {
		return nil
	}

	// it's time to register the group aggregation endpoint
	groupPath := "/apis/" + apiService.Spec.Group
	groupDiscoveryHandler := &apiGroupHandler{
		codecs:    aggregatorscheme.Codecs,
		groupName: apiService.Spec.Group,
		lister:    s.lister,
		delegate:  s.delegateHandler,
	}
	// aggregation is protected
	s.GenericAPIServer.Handler.NonGoRestfulMux.Handle(groupPath, groupDiscoveryHandler)
	s.GenericAPIServer.Handler.NonGoRestfulMux.UnlistedHandle(groupPath+"/", groupDiscoveryHandler)
	s.handledGroups.Insert(apiService.Spec.Group)
	return nil
}