func (c *client) Run()

// Run is for running the device plugin gRPC client.
func (c *client) Run() {
	stream, err := c.client.ListAndWatch(context.Background(), &api.Empty{})
	if err != nil {
		klog.ErrorS(err, "ListAndWatch ended unexpectedly for device plugin", "resource", c.resource)
		return
	}

	for {
		response, err := stream.Recv()
		if err != nil {
			klog.ErrorS(err, "ListAndWatch ended unexpectedly for device plugin", "resource", c.resource)
			return
		}
		klog.V(2).InfoS("State pushed for device plugin", "resource", c.resource, "resourceCapacity", len(response.Devices))
		c.handler.PluginListAndWatchReceiver(c.resource, response)
	}
}


func (c *client) Disconnect() error

// Disconnect is for closing gRPC connection between device manager and device plugin.
func (c *client) Disconnect() error {
	c.mutex.Lock()
	if c.grpc != nil {
		if err := c.grpc.Close(); err != nil {
			klog.V(2).ErrorS(err, "Failed to close grcp connection", "resource", c.Resource())
		}
		c.grpc = nil
	}
	c.mutex.Unlock()
	c.handler.PluginDisconnected(c.resource)
	return nil
}


func (c *client) Resource() string

func (c *client) Resource() string {
	return c.resource
}


func (c *client) API() api.DevicePluginClient {
	return c.client
}


func (c *client) SocketPath() string

func (c *client) SocketPath() string {
	return c.socket
}


// dial establishes the gRPC communication with the registered device plugin. https://godoc.org/google.golang.org/grpc#Dial
func dial(unixSocketPath string) (api.DevicePluginClient, *grpc.ClientConn, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	c, err := grpc.DialContext(ctx, unixSocketPath,
		grpc.WithAuthority("localhost"),
		grpc.WithTransportCredentials(insecure.NewCredentials()),
		grpc.WithBlock(),
		grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
			return (&net.Dialer{}).DialContext(ctx, "unix", addr)
		}),
	)

	if err != nil {
		return nil, nil, fmt.Errorf(errFailedToDialDevicePlugin+" %v", err)
	}

	return api.NewDevicePluginClient(c), c, nil
}


func (s *server) GetPluginHandler() cache.PluginHandler {
	if f, err := os.Create(s.socketDir + "DEPRECATION"); err != nil {
		klog.ErrorS(err, "Failed to create deprecation file at socket dir", "path", s.socketDir)
	} else {
		f.Close()
		klog.V(4).InfoS("Created deprecation file", "path", f.Name())
	}
	return s
}


func (s *server) RegisterPlugin(pluginName string, endpoint string, versions []string) error {
	klog.V(2).InfoS("Registering plugin at endpoint", "plugin", pluginName, "endpoint", endpoint)
	return s.connectClient(pluginName, endpoint)
}


func (s *server) DeRegisterPlugin(pluginName string) {
	klog.V(2).InfoS("Deregistering plugin", "plugin", pluginName)
	client := s.getClient(pluginName)
	if client != nil {
		s.disconnectClient(pluginName, client)
	}
}


func (s *server) ValidatePlugin(pluginName string, endpoint string, versions []string) error {
	klog.V(2).InfoS("Got plugin at endpoint with versions", "plugin", pluginName, "endpoint", endpoint, "versions", versions)

	if !s.isVersionCompatibleWithPlugin(versions...) {
		return fmt.Errorf("manager version, %s, is not among plugin supported versions %v", api.Version, versions)
	}

	if !v1helper.IsExtendedResourceName(core.ResourceName(pluginName)) {
		return fmt.Errorf("invalid name of device plugin socket: %s", fmt.Sprintf(errInvalidResourceName, pluginName))
	}

	return nil
}


func (s *server) connectClient(name string, socketPath string) error {
	c := NewPluginClient(name, socketPath, s.chandler)

	s.registerClient(name, c)
	if err := c.Connect(); err != nil {
		s.deregisterClient(name)
		klog.ErrorS(err, "Failed to connect to new client", "resource", name)
		return err
	}

	go func() {
		s.runClient(name, c)
	}()

	return nil
}


func (s *server) disconnectClient(name string, c Client) error {
	s.deregisterClient(name)
	return c.Disconnect()
}


func (s *server) registerClient(name string, c Client) {
	s.mutex.Lock()
	defer s.mutex.Unlock()

	s.clients[name] = c
	klog.V(2).InfoS("Registered client", "name", name)
}


func (s *server) deregisterClient(name string) {
	s.mutex.Lock()
	defer s.mutex.Unlock()

	delete(s.clients, name)
	klog.V(2).InfoS("Deregistered client", "name", name)
}


func (s *server) runClient(name string, c Client) {
	c.Run()

	c = s.getClient(name)
	if c == nil {
		return
	}

	if err := s.disconnectClient(name, c); err != nil {
		klog.V(2).InfoS("Unable to disconnect client", "resource", name, "client", c, "err", err)
	}
}


func (s *server) getClient(name string) Client {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	return s.clients[name]
}


// NewServer returns an initialized device plugin registration server.
func NewServer(socketPath string, rh RegistrationHandler, ch ClientHandler) (Server, error) {
	if socketPath == "" || !filepath.IsAbs(socketPath) {
		return nil, fmt.Errorf(errBadSocket+" %s", socketPath)
	}

	dir, name := filepath.Split(socketPath)

	klog.V(2).InfoS("Creating device plugin registration server", "version", api.Version, "socket", socketPath)
	s := &server{
		socketName: name,
		socketDir:  dir,
		rhandler:   rh,
		chandler:   ch,
		clients:    make(map[string]Client),
	}

	return s, nil
}


func (s *server) Start() error

func (s *server) Start() error {
	klog.V(2).InfoS("Starting device plugin registration server")

	if err := os.MkdirAll(s.socketDir, 0750); err != nil {
		klog.ErrorS(err, "Failed to create the device plugin socket directory", "directory", s.socketDir)
		return err
	}

	if selinux.GetEnabled() {
		if err := selinux.SetFileLabel(s.socketDir, config.KubeletPluginsDirSELinuxLabel); err != nil {
			klog.InfoS("Unprivileged containerized plugins might not work. Could not set selinux context on socket dir", "path", s.socketDir, "err", err)
		}
	}

	// For now we leave cleanup of the *entire* directory up to the Handler
	// (even though we should in theory be able to just wipe the whole directory)
	// because the Handler stores its checkpoint file (amongst others) in here.
	if err := s.rhandler.CleanupPluginDirectory(s.socketDir); err != nil {
		klog.ErrorS(err, "Failed to cleanup the device plugin directory", "directory", s.socketDir)
		return err
	}

	ln, err := net.Listen("unix", s.SocketPath())
	if err != nil {
		klog.ErrorS(err, "Failed to listen to socket while starting device plugin registry")
		return err
	}

	s.wg.Add(1)
	s.grpc = grpc.NewServer([]grpc.ServerOption{}...)

	api.RegisterRegistrationServer(s.grpc, s)
	go func() {
		defer s.wg.Done()
		s.grpc.Serve(ln)
	}()

	return nil
}


func (s *server) Stop() error

func (s *server) Stop() error {
	s.visitClients(func(r string, c Client) {
		if err := s.disconnectClient(r, c); err != nil {
			klog.InfoS("Error disconnecting device plugin client", "resourceName", r, "err", err)
		}
	})

	s.mutex.Lock()
	defer s.mutex.Unlock()

	if s.grpc == nil {
		return nil
	}

	s.grpc.Stop()
	s.wg.Wait()
	s.grpc = nil

	return nil
}


func (s *server) SocketPath() string

func (s *server) SocketPath() string {
	return filepath.Join(s.socketDir, s.socketName)
}


func (s *server) Register(ctx context.Context, r *api.RegisterRequest) (*api.Empty, error) {
	klog.InfoS("Got registration request from device plugin with resource", "resourceName", r.ResourceName)
	metrics.DevicePluginRegistrationCount.WithLabelValues(r.ResourceName).Inc()

	if !s.isVersionCompatibleWithPlugin(r.Version) {
		err := fmt.Errorf(errUnsupportedVersion, r.Version, api.SupportedVersions)
		klog.InfoS("Bad registration request from device plugin with resource", "resourceName", r.ResourceName, "err", err)
		return &api.Empty{}, err
	}

	if !v1helper.IsExtendedResourceName(core.ResourceName(r.ResourceName)) {
		err := fmt.Errorf(errInvalidResourceName, r.ResourceName)
		klog.InfoS("Bad registration request from device plugin", "err", err)
		return &api.Empty{}, err
	}

	if err := s.connectClient(r.ResourceName, filepath.Join(s.socketDir, r.Endpoint)); err != nil {
		klog.InfoS("Error connecting to device plugin client", "err", err)
		return &api.Empty{}, err
	}

	return &api.Empty{}, nil
}


func (s *server) isVersionCompatibleWithPlugin(versions ...string) bool {
	// TODO(vikasc): Currently this is fine as we only have a single supported version. When we do need to support
	// multiple versions in the future, we may need to extend this function to return a supported version.
	// E.g., say kubelet supports v1beta1 and v1beta2, and we get v1alpha1 and v1beta1 from a device plugin,
	// this function should return v1beta1
	for _, version := range versions {
		for _, supportedVersion := range api.SupportedVersions {
			if version == supportedVersion {
				return true
			}
		}
	}
	return false
}


func (s *server) visitClients(visit func(r string, c Client)) {
	s.mutex.Lock()
	for r, c := range s.clients {
		s.mutex.Unlock()
		visit(r, c)
		s.mutex.Lock()
	}
	s.mutex.Unlock()
}


func defaultGetPreferredAllocFunc(r *pluginapi.PreferredAllocationRequest, devs map[string]pluginapi.Device) (*pluginapi.PreferredAllocationResponse, error) {
	var response pluginapi.PreferredAllocationResponse

	return &response, nil
}


func defaultAllocFunc(r *pluginapi.AllocateRequest, devs map[string]pluginapi.Device) (*pluginapi.AllocateResponse, error) {
	var response pluginapi.AllocateResponse

	return &response, nil
}


// NewDevicePluginStub returns an initialized DevicePlugin Stub.
func NewDevicePluginStub(devs []*pluginapi.Device, socket string, name string, preStartContainerFlag bool, getPreferredAllocationFlag bool) *Stub {
	return &Stub{
		devs:                       devs,
		socket:                     socket,
		resourceName:               name,
		preStartContainerFlag:      preStartContainerFlag,
		getPreferredAllocationFlag: getPreferredAllocationFlag,

		stop:   make(chan interface{}),
		update: make(chan []*pluginapi.Device),

		allocFunc:             defaultAllocFunc,
		getPreferredAllocFunc: defaultGetPreferredAllocFunc,
	}
}


// SetGetPreferredAllocFunc sets allocFunc of the device plugin
func (m *Stub) SetGetPreferredAllocFunc(f stubGetPreferredAllocFunc) {
	m.getPreferredAllocFunc = f
}


// SetAllocFunc sets allocFunc of the device plugin
func (m *Stub) SetAllocFunc(f stubAllocFunc) {
	m.allocFunc = f
}


func (m *Stub) Start() error

// Start starts the gRPC server of the device plugin. Can only
// be called once.
func (m *Stub) Start() error {
	err := m.cleanup()
	if err != nil {
		return err
	}

	sock, err := net.Listen("unix", m.socket)
	if err != nil {
		return err
	}

	m.wg.Add(1)
	m.server = grpc.NewServer([]grpc.ServerOption{}...)
	pluginapi.RegisterDevicePluginServer(m.server, m)
	watcherapi.RegisterRegistrationServer(m.server, m)

	go func() {
		defer m.wg.Done()
		m.server.Serve(sock)
	}()

	var lastDialErr error
	wait.PollImmediate(1*time.Second, 10*time.Second, func() (bool, error) {
		var conn *grpc.ClientConn
		_, conn, lastDialErr = dial(m.socket)
		if lastDialErr != nil {
			return false, nil
		}
		conn.Close()
		return true, nil
	})
	if lastDialErr != nil {
		return lastDialErr
	}

	klog.InfoS("Starting to serve on socket", "socket", m.socket)
	return nil
}


func (m *Stub) Stop() error

// Stop stops the gRPC server. Can be called without a prior Start
// and more than once. Not safe to be called concurrently by different
// goroutines!
func (m *Stub) Stop() error {
	if m.server == nil {
		return nil
	}
	m.server.Stop()
	m.wg.Wait()
	m.server = nil
	close(m.stop) // This prevents re-starting the server.

	return m.cleanup()
}


// GetInfo is the RPC which return pluginInfo
func (m *Stub) GetInfo(ctx context.Context, req *watcherapi.InfoRequest) (*watcherapi.PluginInfo, error) {
	klog.InfoS("GetInfo")
	return &watcherapi.PluginInfo{
		Type:              watcherapi.DevicePlugin,
		Name:              m.resourceName,
		Endpoint:          m.endpoint,
		SupportedVersions: []string{pluginapi.Version}}, nil
}


// NotifyRegistrationStatus receives the registration notification from watcher
func (m *Stub) NotifyRegistrationStatus(ctx context.Context, status *watcherapi.RegistrationStatus) (*watcherapi.RegistrationStatusResponse, error) {
	if m.registrationStatus != nil {
		m.registrationStatus <- *status
	}
	if !status.PluginRegistered {
		klog.InfoS("Registration failed", "err", status.Error)
	}
	return &watcherapi.RegistrationStatusResponse{}, nil
}


// Register registers the device plugin for the given resourceName with Kubelet.
func (m *Stub) Register(kubeletEndpoint, resourceName string, pluginSockDir string) error {
	if pluginSockDir != "" {
		if _, err := os.Stat(pluginSockDir + "DEPRECATION"); err == nil {
			klog.InfoS("Deprecation file found. Skip registration")
			return nil
		}
	}
	klog.InfoS("Deprecation file not found. Invoke registration")
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	conn, err := grpc.DialContext(ctx, kubeletEndpoint,
		grpc.WithTransportCredentials(insecure.NewCredentials()),
		grpc.WithBlock(),
		grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
			return (&net.Dialer{}).DialContext(ctx, "unix", addr)
		}))
	if err != nil {
		return err
	}
	defer conn.Close()
	client := pluginapi.NewRegistrationClient(conn)
	reqt := &pluginapi.RegisterRequest{
		Version:      pluginapi.Version,
		Endpoint:     filepath.Base(m.socket),
		ResourceName: resourceName,
		Options: &pluginapi.DevicePluginOptions{
			PreStartRequired:                m.preStartContainerFlag,
			GetPreferredAllocationAvailable: m.getPreferredAllocationFlag,
		},
	}

	_, err = client.Register(context.Background(), reqt)
	return err
}


// GetDevicePluginOptions returns DevicePluginOptions settings for the device plugin.
func (m *Stub) GetDevicePluginOptions(ctx context.Context, e *pluginapi.Empty) (*pluginapi.DevicePluginOptions, error) {
	options := &pluginapi.DevicePluginOptions{
		PreStartRequired:                m.preStartContainerFlag,
		GetPreferredAllocationAvailable: m.getPreferredAllocationFlag,
	}
	return options, nil
}


// PreStartContainer resets the devices received
func (m *Stub) PreStartContainer(ctx context.Context, r *pluginapi.PreStartContainerRequest) (*pluginapi.PreStartContainerResponse, error) {
	klog.InfoS("PreStartContainer", "request", r)
	return &pluginapi.PreStartContainerResponse{}, nil
}


// ListAndWatch lists devices and update that list according to the Update call
func (m *Stub) ListAndWatch(e *pluginapi.Empty, s pluginapi.DevicePlugin_ListAndWatchServer) error {
	klog.InfoS("ListAndWatch")

	s.Send(&pluginapi.ListAndWatchResponse{Devices: m.devs})

	for {
		select {
		case <-m.stop:
			return nil
		case updated := <-m.update:
			s.Send(&pluginapi.ListAndWatchResponse{Devices: updated})
		}
	}
}


// Update allows the device plugin to send new devices through ListAndWatch
func (m *Stub) Update(devs []*pluginapi.Device) {
	m.update <- devs
}


// GetPreferredAllocation gets the preferred allocation from a set of available devices
func (m *Stub) GetPreferredAllocation(ctx context.Context, r *pluginapi.PreferredAllocationRequest) (*pluginapi.PreferredAllocationResponse, error) {
	klog.InfoS("GetPreferredAllocation", "request", r)

	devs := make(map[string]pluginapi.Device)

	for _, dev := range m.devs {
		devs[dev.ID] = *dev
	}

	return m.getPreferredAllocFunc(r, devs)
}


// Allocate does a mock allocation
func (m *Stub) Allocate(ctx context.Context, r *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) {
	klog.InfoS("Allocate", "request", r)

	devs := make(map[string]pluginapi.Device)

	for _, dev := range m.devs {
		devs[dev.ID] = *dev
	}

	return m.allocFunc(r, devs)
}


func (m *Stub) cleanup() error

func (m *Stub) cleanup() error {
	if err := os.Remove(m.socket); err != nil && !os.IsNotExist(err) {
		return err
	}

	return nil
}


func newPodDevices() *podDevices

// NewPodDevices is a function that returns object of podDevices type with its own guard
// RWMutex and a map where key is a pod UID and value contains
// container devices information of type containerDevices.
func newPodDevices() *podDevices {
	return &podDevices{devs: make(map[string]containerDevices)}
}


func (pdev *podDevices) pods() sets.String {
	pdev.RLock()
	defer pdev.RUnlock()
	ret := sets.NewString()
	for k := range pdev.devs {
		ret.Insert(k)
	}
	return ret
}


func (pdev *podDevices) size() int

func (pdev *podDevices) size() int {
	pdev.RLock()
	defer pdev.RUnlock()
	return len(pdev.devs)
}


func (pdev *podDevices) hasPod(podUID string) bool {
	pdev.RLock()
	defer pdev.RUnlock()
	_, podExists := pdev.devs[podUID]
	return podExists
}


func (pdev *podDevices) insert(podUID, contName, resource string, devices checkpoint.DevicesPerNUMA, resp *pluginapi.ContainerAllocateResponse) {
	pdev.Lock()
	defer pdev.Unlock()
	if _, podExists := pdev.devs[podUID]; !podExists {
		pdev.devs[podUID] = make(containerDevices)
	}
	if _, contExists := pdev.devs[podUID][contName]; !contExists {
		pdev.devs[podUID][contName] = make(resourceAllocateInfo)
	}
	pdev.devs[podUID][contName][resource] = deviceAllocateInfo{
		deviceIds: devices,
		allocResp: resp,
	}
}


func (pdev *podDevices) delete(pods []string) {
	pdev.Lock()
	defer pdev.Unlock()
	for _, uid := range pods {
		delete(pdev.devs, uid)
	}
}


// Returns list of device Ids allocated to the given pod for the given resource.
// Returns nil if we don't have cached state for the given <podUID, resource>.
func (pdev *podDevices) podDevices(podUID, resource string) sets.String {
	pdev.RLock()
	defer pdev.RUnlock()

	ret := sets.NewString()
	for contName := range pdev.devs[podUID] {
		ret = ret.Union(pdev.containerDevices(podUID, contName, resource))
	}
	return ret
}


// Returns list of device Ids allocated to the given container for the given resource.
// Returns nil if we don't have cached state for the given <podUID, contName, resource>.
func (pdev *podDevices) containerDevices(podUID, contName, resource string) sets.String {
	pdev.RLock()
	defer pdev.RUnlock()
	if _, podExists := pdev.devs[podUID]; !podExists {
		return nil
	}
	if _, contExists := pdev.devs[podUID][contName]; !contExists {
		return nil
	}
	devs, resourceExists := pdev.devs[podUID][contName][resource]
	if !resourceExists {
		return nil
	}
	return devs.deviceIds.Devices()
}


// Populates allocatedResources with the device resources allocated to the specified <podUID, contName>.
func (pdev *podDevices) addContainerAllocatedResources(podUID, contName string, allocatedResources map[string]sets.String) {
	pdev.RLock()
	defer pdev.RUnlock()
	containers, exists := pdev.devs[podUID]
	if !exists {
		return
	}
	resources, exists := containers[contName]
	if !exists {
		return
	}
	for resource, devices := range resources {
		allocatedResources[resource] = allocatedResources[resource].Union(devices.deviceIds.Devices())
	}
}


// Removes the device resources allocated to the specified <podUID, contName> from allocatedResources.
func (pdev *podDevices) removeContainerAllocatedResources(podUID, contName string, allocatedResources map[string]sets.String) {
	pdev.RLock()
	defer pdev.RUnlock()
	containers, exists := pdev.devs[podUID]
	if !exists {
		return
	}
	resources, exists := containers[contName]
	if !exists {
		return
	}
	for resource, devices := range resources {
		allocatedResources[resource] = allocatedResources[resource].Difference(devices.deviceIds.Devices())
	}
}


// Returns all of devices allocated to the pods being tracked, keyed by resourceName.
func (pdev *podDevices) devices() map[string]sets.String {
	ret := make(map[string]sets.String)
	pdev.RLock()
	defer pdev.RUnlock()
	for _, containerDevices := range pdev.devs {
		for _, resources := range containerDevices {
			for resource, devices := range resources {
				if _, exists := ret[resource]; !exists {
					ret[resource] = sets.NewString()
				}
				if devices.allocResp != nil {
					ret[resource] = ret[resource].Union(devices.deviceIds.Devices())
				}
			}
		}
	}
	return ret
}


// Turns podDevices to checkpointData.
func (pdev *podDevices) toCheckpointData() []checkpoint.PodDevicesEntry {
	var data []checkpoint.PodDevicesEntry
	pdev.RLock()
	defer pdev.RUnlock()
	for podUID, containerDevices := range pdev.devs {
		for conName, resources := range containerDevices {
			for resource, devices := range resources {
				if devices.allocResp == nil {
					klog.ErrorS(nil, "Can't marshal allocResp, allocation response is missing", "podUID", podUID, "containerName", conName, "resourceName", resource)
					continue
				}

				allocResp, err := devices.allocResp.Marshal()
				if err != nil {
					klog.ErrorS(err, "Can't marshal allocResp", "podUID", podUID, "containerName", conName, "resourceName", resource)
					continue
				}
				data = append(data, checkpoint.PodDevicesEntry{
					PodUID:        podUID,
					ContainerName: conName,
					ResourceName:  resource,
					DeviceIDs:     devices.deviceIds,
					AllocResp:     allocResp})
			}
		}
	}
	return data
}


// Populates podDevices from the passed in checkpointData.
func (pdev *podDevices) fromCheckpointData(data []checkpoint.PodDevicesEntry) {
	for _, entry := range data {
		klog.V(2).InfoS("Get checkpoint entry",
			"podUID", entry.PodUID, "containerName", entry.ContainerName,
			"resourceName", entry.ResourceName, "deviceIDs", entry.DeviceIDs, "allocated", entry.AllocResp)
		allocResp := &pluginapi.ContainerAllocateResponse{}
		err := allocResp.Unmarshal(entry.AllocResp)
		if err != nil {
			klog.ErrorS(err, "Can't unmarshal allocResp", "podUID", entry.PodUID, "containerName", entry.ContainerName, "resourceName", entry.ResourceName)
			continue
		}
		pdev.insert(entry.PodUID, entry.ContainerName, entry.ResourceName, entry.DeviceIDs, allocResp)
	}
}


// Returns combined container runtime settings to consume the container's allocated devices.
func (pdev *podDevices) deviceRunContainerOptions(podUID, contName string) *DeviceRunContainerOptions {
	pdev.RLock()
	defer pdev.RUnlock()

	containers, exists := pdev.devs[podUID]
	if !exists {
		return nil
	}
	resources, exists := containers[contName]
	if !exists {
		return nil
	}
	opts := &DeviceRunContainerOptions{}
	// Maps to detect duplicate settings.
	devsMap := make(map[string]string)
	mountsMap := make(map[string]string)
	envsMap := make(map[string]string)
	annotationsMap := make(map[string]string)
	// Loops through AllocationResponses of all cached device resources.
	for _, devices := range resources {
		resp := devices.allocResp
		// Each Allocate response has the following artifacts.
		// Environment variables
		// Mount points
		// Device files
		// Container annotations
		// These artifacts are per resource per container.
		// Updates RunContainerOptions.Envs.
		for k, v := range resp.Envs {
			if e, ok := envsMap[k]; ok {
				klog.V(4).InfoS("Skip existing env", "envKey", k, "envValue", v)
				if e != v {
					klog.ErrorS(nil, "Environment variable has conflicting setting", "envKey", k, "expected", v, "got", e)
				}
				continue
			}
			klog.V(4).InfoS("Add env", "envKey", k, "envValue", v)
			envsMap[k] = v
			opts.Envs = append(opts.Envs, kubecontainer.EnvVar{Name: k, Value: v})
		}

		// Updates RunContainerOptions.Devices.
		for _, dev := range resp.Devices {
			if d, ok := devsMap[dev.ContainerPath]; ok {
				klog.V(4).InfoS("Skip existing device", "containerPath", dev.ContainerPath, "hostPath", dev.HostPath)
				if d != dev.HostPath {
					klog.ErrorS(nil, "Container device has conflicting mapping host devices",
						"containerPath", dev.ContainerPath, "got", d, "expected", dev.HostPath)
				}
				continue
			}
			klog.V(4).InfoS("Add device", "containerPath", dev.ContainerPath, "hostPath", dev.HostPath)
			devsMap[dev.ContainerPath] = dev.HostPath
			opts.Devices = append(opts.Devices, kubecontainer.DeviceInfo{
				PathOnHost:      dev.HostPath,
				PathInContainer: dev.ContainerPath,
				Permissions:     dev.Permissions,
			})
		}

		// Updates RunContainerOptions.Mounts.
		for _, mount := range resp.Mounts {
			if m, ok := mountsMap[mount.ContainerPath]; ok {
				klog.V(4).InfoS("Skip existing mount", "containerPath", mount.ContainerPath, "hostPath", mount.HostPath)
				if m != mount.HostPath {
					klog.ErrorS(nil, "Container mount has conflicting mapping host mounts",
						"containerPath", mount.ContainerPath, "conflictingPath", m, "hostPath", mount.HostPath)
				}
				continue
			}
			klog.V(4).InfoS("Add mount", "containerPath", mount.ContainerPath, "hostPath", mount.HostPath)
			mountsMap[mount.ContainerPath] = mount.HostPath
			opts.Mounts = append(opts.Mounts, kubecontainer.Mount{
				Name:          mount.ContainerPath,
				ContainerPath: mount.ContainerPath,
				HostPath:      mount.HostPath,
				ReadOnly:      mount.ReadOnly,
				// TODO: This may need to be part of Device plugin API.
				SELinuxRelabel: false,
			})
		}

		// Updates for Annotations
		for k, v := range resp.Annotations {
			if e, ok := annotationsMap[k]; ok {
				klog.V(4).InfoS("Skip existing annotation", "annotationKey", k, "annotationValue", v)
				if e != v {
					klog.ErrorS(nil, "Annotation has conflicting setting", "annotationKey", k, "expected", e, "got", v)
				}
				continue
			}
			klog.V(4).InfoS("Add annotation", "annotationKey", k, "annotationValue", v)
			annotationsMap[k] = v
			opts.Annotations = append(opts.Annotations, kubecontainer.Annotation{Name: k, Value: v})
		}
	}
	return opts
}


// getContainerDevices returns the devices assigned to the provided container for all ResourceNames
func (pdev *podDevices) getContainerDevices(podUID, contName string) ResourceDeviceInstances {
	pdev.RLock()
	defer pdev.RUnlock()

	if _, podExists := pdev.devs[podUID]; !podExists {
		return nil
	}
	if _, contExists := pdev.devs[podUID][contName]; !contExists {
		return nil
	}
	resDev := NewResourceDeviceInstances()
	for resource, allocateInfo := range pdev.devs[podUID][contName] {
		if len(allocateInfo.deviceIds) == 0 {
			continue
		}
		devicePluginMap := make(map[string]pluginapi.Device)
		for numaid, devlist := range allocateInfo.deviceIds {
			for _, devID := range devlist {
				var topology *pluginapi.TopologyInfo
				if numaid != nodeWithoutTopology {
					NUMANodes := []*pluginapi.NUMANode{{ID: numaid}}
					if pDev, ok := devicePluginMap[devID]; ok && pDev.Topology != nil {
						if nodes := pDev.Topology.GetNodes(); nodes != nil {
							NUMANodes = append(NUMANodes, nodes...)
						}
					}

					// ID and Healthy are not relevant here.
					topology = &pluginapi.TopologyInfo{Nodes: NUMANodes}
				}
				devicePluginMap[devID] = pluginapi.Device{
					Topology: topology,
				}
			}
		}
		resDev[resource] = devicePluginMap
	}
	return resDev
}


// NewResourceDeviceInstances returns a new ResourceDeviceInstances
func NewResourceDeviceInstances() ResourceDeviceInstances {
	return make(ResourceDeviceInstances)
}


// Clone returns a clone of ResourceDeviceInstances
func (rdev ResourceDeviceInstances) Clone() ResourceDeviceInstances {
	clone := NewResourceDeviceInstances()
	for resourceName, resourceDevs := range rdev {
		clone[resourceName] = make(map[string]pluginapi.Device)
		for devID, dev := range resourceDevs {
			clone[resourceName][devID] = dev
		}
	}
	return clone
}


// Filter takes a condition set expressed as map[string]sets.String and returns a new
// ResourceDeviceInstances with only the devices matching the condition set.
func (rdev ResourceDeviceInstances) Filter(cond map[string]sets.String) ResourceDeviceInstances {
	filtered := NewResourceDeviceInstances()
	for resourceName, filterIDs := range cond {
		if _, exists := rdev[resourceName]; !exists {
			continue
		}
		filtered[resourceName] = DeviceInstances{}
		for instanceID, instance := range rdev[resourceName] {
			if filterIDs.Has(instanceID) {
				filtered[resourceName][instanceID] = instance
			}
		}
	}
	return filtered
}


// GetTopologyHints implements the TopologyManager HintProvider Interface which
// ensures the Device Manager is consulted when Topology Aware Hints for each
// container are created.
func (m *ManagerImpl) GetTopologyHints(pod *v1.Pod, container *v1.Container) map[string][]topologymanager.TopologyHint {
	// The pod is during the admission phase. We need to save the pod to avoid it
	// being cleaned before the admission ended
	m.setPodPendingAdmission(pod)

	// Garbage collect any stranded device resources before providing TopologyHints
	m.UpdateAllocatedDevices()

	// Loop through all device resources and generate TopologyHints for them..
	deviceHints := make(map[string][]topologymanager.TopologyHint)
	accumulatedResourceRequests := m.getContainerDeviceRequest(container)

	m.mutex.Lock()
	defer m.mutex.Unlock()
	for resource, requested := range accumulatedResourceRequests {
		// Only consider devices that actually contain topology information.
		if aligned := m.deviceHasTopologyAlignment(resource); !aligned {
			klog.InfoS("Resource does not have a topology preference", "resource", resource)
			deviceHints[resource] = nil
			continue
		}

		// Short circuit to regenerate the same hints if there are already
		// devices allocated to the Container. This might happen after a
		// kubelet restart, for example.
		allocated := m.podDevices.containerDevices(string(pod.UID), container.Name, resource)
		if allocated.Len() > 0 {
			if allocated.Len() != requested {
				klog.ErrorS(nil, "Resource already allocated to pod with different number than request", "resource", resource, "pod", klog.KObj(pod), "containerName", container.Name, "request", requested, "allocated", allocated.Len())
				deviceHints[resource] = []topologymanager.TopologyHint{}
				continue
			}
			klog.InfoS("Regenerating TopologyHints for resource already allocated to pod", "resource", resource, "pod", klog.KObj(pod), "containerName", container.Name)
			deviceHints[resource] = m.generateDeviceTopologyHints(resource, allocated, sets.String{}, requested)
			continue
		}

		// Get the list of available devices, for which TopologyHints should be generated.
		available := m.getAvailableDevices(resource)
		reusable := m.devicesToReuse[string(pod.UID)][resource]
		if available.Union(reusable).Len() < requested {
			klog.ErrorS(nil, "Unable to generate topology hints: requested number of devices unavailable", "resource", resource, "request", requested, "available", available.Union(reusable).Len())
			deviceHints[resource] = []topologymanager.TopologyHint{}
			continue
		}

		// Generate TopologyHints for this resource given the current
		// request size and the list of available devices.
		deviceHints[resource] = m.generateDeviceTopologyHints(resource, available, reusable, requested)
	}

	return deviceHints
}


// GetPodTopologyHints implements the topologymanager.HintProvider Interface which
// ensures the Device Manager is consulted when Topology Aware Hints for Pod are created.
func (m *ManagerImpl) GetPodTopologyHints(pod *v1.Pod) map[string][]topologymanager.TopologyHint {
	// The pod is during the admission phase. We need to save the pod to avoid it
	// being cleaned before the admission ended
	m.setPodPendingAdmission(pod)

	// Garbage collect any stranded device resources before providing TopologyHints
	m.UpdateAllocatedDevices()

	deviceHints := make(map[string][]topologymanager.TopologyHint)
	accumulatedResourceRequests := m.getPodDeviceRequest(pod)

	m.mutex.Lock()
	defer m.mutex.Unlock()
	for resource, requested := range accumulatedResourceRequests {
		// Only consider devices that actually contain topology information.
		if aligned := m.deviceHasTopologyAlignment(resource); !aligned {
			klog.InfoS("Resource does not have a topology preference", "resource", resource)
			deviceHints[resource] = nil
			continue
		}

		// Short circuit to regenerate the same hints if there are already
		// devices allocated to the Pod. This might happen after a
		// kubelet restart, for example.
		allocated := m.podDevices.podDevices(string(pod.UID), resource)
		if allocated.Len() > 0 {
			if allocated.Len() != requested {
				klog.ErrorS(nil, "Resource already allocated to pod with different number than request", "resource", resource, "pod", klog.KObj(pod), "request", requested, "allocated", allocated.Len())
				deviceHints[resource] = []topologymanager.TopologyHint{}
				continue
			}
			klog.InfoS("Regenerating TopologyHints for resource already allocated to pod", "resource", resource, "pod", klog.KObj(pod))
			deviceHints[resource] = m.generateDeviceTopologyHints(resource, allocated, sets.String{}, requested)
			continue
		}

		// Get the list of available devices, for which TopologyHints should be generated.
		available := m.getAvailableDevices(resource)
		if available.Len() < requested {
			klog.ErrorS(nil, "Unable to generate topology hints: requested number of devices unavailable", "resource", resource, "request", requested, "available", available.Len())
			deviceHints[resource] = []topologymanager.TopologyHint{}
			continue
		}

		// Generate TopologyHints for this resource given the current
		// request size and the list of available devices.
		deviceHints[resource] = m.generateDeviceTopologyHints(resource, available, sets.String{}, requested)
	}

	return deviceHints
}


func (m *ManagerImpl) deviceHasTopologyAlignment(resource string) bool {
	// If any device has Topology NUMANodes available, we assume they care about alignment.
	for _, device := range m.allDevices[resource] {
		if device.Topology != nil && len(device.Topology.Nodes) > 0 {
			return true
		}
	}
	return false
}


func (m *ManagerImpl) getAvailableDevices(resource string) sets.String {
	// Strip all devices in use from the list of healthy ones.
	return m.healthyDevices[resource].Difference(m.allocatedDevices[resource])
}


func (m *ManagerImpl) generateDeviceTopologyHints(resource string, available sets.String, reusable sets.String, request int) []topologymanager.TopologyHint {
	// Initialize minAffinitySize to include all NUMA Nodes
	minAffinitySize := len(m.numaNodes)

	// Iterate through all combinations of NUMA Nodes and build hints from them.
	hints := []topologymanager.TopologyHint{}
	bitmask.IterateBitMasks(m.numaNodes, func(mask bitmask.BitMask) {
		// First, update minAffinitySize for the current request size.
		devicesInMask := 0
		for _, device := range m.allDevices[resource] {
			if mask.AnySet(m.getNUMANodeIds(device.Topology)) {
				devicesInMask++
			}
		}
		if devicesInMask >= request && mask.Count() < minAffinitySize {
			minAffinitySize = mask.Count()
		}

		// Then check to see if all of the reusable devices are part of the bitmask.
		numMatching := 0
		for d := range reusable {
			// Skip the device if it doesn't specify any topology info.
			if m.allDevices[resource][d].Topology == nil {
				continue
			}
			// Otherwise disregard this mask if its NUMANode isn't part of it.
			if !mask.AnySet(m.getNUMANodeIds(m.allDevices[resource][d].Topology)) {
				return
			}
			numMatching++
		}

		// Finally, check to see if enough available devices remain on the
		// current NUMA node combination to satisfy the device request.
		for d := range available {
			if mask.AnySet(m.getNUMANodeIds(m.allDevices[resource][d].Topology)) {
				numMatching++
			}
		}

		// If they don't, then move onto the next combination.
		if numMatching < request {
			return
		}

		// Otherwise, create a new hint from the NUMA mask and add it to the
		// list of hints.  We set all hint preferences to 'false' on the first
		// pass through.
		hints = append(hints, topologymanager.TopologyHint{
			NUMANodeAffinity: mask,
			Preferred:        false,
		})
	})

	// Loop back through all hints and update the 'Preferred' field based on
	// counting the number of bits sets in the affinity mask and comparing it
	// to the minAffinity. Only those with an equal number of bits set will be
	// considered preferred.
	for i := range hints {
		if hints[i].NUMANodeAffinity.Count() == minAffinitySize {
			hints[i].Preferred = true
		}
	}

	return hints
}


func (m *ManagerImpl) getNUMANodeIds(topology *pluginapi.TopologyInfo) []int {
	if topology == nil {
		return nil
	}
	var ids []int
	for _, n := range topology.Nodes {
		ids = append(ids, int(n.ID))
	}
	return ids
}


func (m *ManagerImpl) getPodDeviceRequest(pod *v1.Pod) map[string]int {
	// for these device plugin resources, requests == limits
	limits := resource.PodLimits(pod, resource.PodResourcesOptions{
		ExcludeOverhead: true,
	})
	podRequests := make(map[string]int)
	for resourceName, quantity := range limits {
		if !m.isDevicePluginResource(string(resourceName)) {
			continue
		}
		podRequests[string(resourceName)] = int(quantity.Value())
	}
	return podRequests
}


func (m *ManagerImpl) getContainerDeviceRequest(container *v1.Container) map[string]int {
	containerRequests := make(map[string]int)
	for resourceObj, requestedObj := range container.Resources.Limits {
		resource := string(resourceObj)
		requested := int(requestedObj.Value())
		if !m.isDevicePluginResource(resource) {
			continue
		}
		containerRequests[resource] = requested
	}
	return containerRequests
}


func generateCDIAnnotations(

// generate container annotations using CDI UpdateAnnotations API.
func generateCDIAnnotations(
	claimUID types.UID,
	driverName string,
	cdiDevices []string,
) ([]kubecontainer.Annotation, error) {
	annotations, err := updateAnnotations(map[string]string{}, driverName, string(claimUID), cdiDevices)
	if err != nil {
		return nil, fmt.Errorf("can't generate CDI annotations: %+v", err)
	}

	kubeAnnotations := []kubecontainer.Annotation{}
	for key, value := range annotations {
		kubeAnnotations = append(kubeAnnotations, kubecontainer.Annotation{Name: key, Value: value})
	}

	return kubeAnnotations, nil
}


// updateAnnotations updates annotations with a plugin-specific CDI device
// injection request for the given devices. Upon any error a non-nil error
// is returned and annotations are left intact. By convention plugin should
// be in the format of "vendor.device-type".
func updateAnnotations(annotations map[string]string, plugin string, deviceID string, devices []string) (map[string]string, error) {
	key, err := annotationKey(plugin, deviceID)
	if err != nil {
		return annotations, fmt.Errorf("CDI annotation failed: %v", err)
	}
	if _, ok := annotations[key]; ok {
		return annotations, fmt.Errorf("CDI annotation failed, key %q used", key)
	}
	value, err := annotationValue(devices)
	if err != nil {
		return annotations, fmt.Errorf("CDI annotation failed: %v", err)
	}

	if annotations == nil {
		annotations = make(map[string]string)
	}
	annotations[key] = value

	return annotations, nil
}


// annotationKey returns a unique annotation key for an device allocation
// by a K8s device plugin. pluginName should be in the format of
// "vendor.device-type". deviceID is the ID of the device the plugin is
// allocating. It is used to make sure that the generated key is unique
// even if multiple allocations by a single plugin needs to be annotated.
func annotationKey(pluginName, deviceID string) (string, error) {
	const maxNameLen = 63

	if pluginName == "" {
		return "", errors.New("invalid plugin name, empty")
	}
	if deviceID == "" {
		return "", errors.New("invalid deviceID, empty")
	}

	name := pluginName + "_" + strings.ReplaceAll(deviceID, "/", "_")

	if len(name) > maxNameLen {
		return "", fmt.Errorf("invalid plugin+deviceID %q, too long", name)
	}

	if c := rune(name[0]); !isAlphaNumeric(c) {
		return "", fmt.Errorf("invalid name %q, first '%c' should be alphanumeric", name, c)
	}
	if len(name) > 2 {
		for _, c := range name[1 : len(name)-1] {
			switch {
			case isAlphaNumeric(c):
			case c == '_' || c == '-' || c == '.':
			default:
				return "", fmt.Errorf("invalid name %q, invalid charcter '%c'", name, c)
			}
		}
	}
	if c := rune(name[len(name)-1]); !isAlphaNumeric(c) {
		return "", fmt.Errorf("invalid name %q, last '%c' should be alphanumeric", name, c)
	}

	return annotationPrefix + name, nil
}


// annotationValue returns an annotation value for the given devices.
func annotationValue(devices []string) (string, error) {
	value, sep := "", ""
	for _, d := range devices {
		if _, _, _, err := parseQualifiedName(d); err != nil {
			return "", err
		}
		value += sep + d
		sep = ","
	}

	return value, nil
}


// parseQualifiedName splits a qualified name into device vendor, class,
// and name. If the device fails to parse as a qualified name, or if any
// of the split components fail to pass syntax validation, vendor and
// class are returned as empty, together with the verbatim input as the
// name and an error describing the reason for failure.
func parseQualifiedName(device string) (string, string, string, error) {
	vendor, class, name := parseDevice(device)

	if vendor == "" {
		return "", "", device, fmt.Errorf("unqualified device %q, missing vendor", device)
	}
	if class == "" {
		return "", "", device, fmt.Errorf("unqualified device %q, missing class", device)
	}
	if name == "" {
		return "", "", device, fmt.Errorf("unqualified device %q, missing device name", device)
	}

	if err := validateVendorName(vendor); err != nil {
		return "", "", device, fmt.Errorf("invalid device %q: %v", device, err)
	}
	if err := validateClassName(class); err != nil {
		return "", "", device, fmt.Errorf("invalid device %q: %v", device, err)
	}
	if err := validateDeviceName(name); err != nil {
		return "", "", device, fmt.Errorf("invalid device %q: %v", device, err)
	}

	return vendor, class, name, nil
}


// parseDevice tries to split a device name into vendor, class, and name.
// If this fails, for instance in the case of unqualified device names,
// parseDevice returns an empty vendor and class together with name set
// to the verbatim input.
func parseDevice(device string) (string, string, string) {
	if device == "" || device[0] == '/' {
		return "", "", device
	}

	parts := strings.SplitN(device, "=", 2)
	if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
		return "", "", device
	}

	name := parts[1]
	vendor, class := parseQualifier(parts[0])
	if vendor == "" {
		return "", "", device
	}

	return vendor, class, name
}


// parseQualifier splits a device qualifier into vendor and class.
// The syntax for a device qualifier is
//
//	"<vendor>/<class>"
//
// If parsing fails, an empty vendor and the class set to the
// verbatim input is returned.
func parseQualifier(kind string) (string, string) {
	parts := strings.SplitN(kind, "/", 2)
	if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
		return "", kind
	}
	return parts[0], parts[1]
}


// validateVendorName checks the validity of a vendor name.
// A vendor name may contain the following ASCII characters:
//   - upper- and lowercase letters ('A'-'Z', 'a'-'z')
//   - digits ('0'-'9')
//   - underscore, dash, and dot ('_', '-', and '.')
func validateVendorName(vendor string) error {
	if vendor == "" {
		return fmt.Errorf("invalid (empty) vendor name")
	}
	if !isLetter(rune(vendor[0])) {
		return fmt.Errorf("invalid vendor %q, should start with letter", vendor)
	}
	for _, c := range string(vendor[1 : len(vendor)-1]) {
		switch {
		case isAlphaNumeric(c):
		case c == '_' || c == '-' || c == '.':
		default:
			return fmt.Errorf("invalid character '%c' in vendor name %q",
				c, vendor)
		}
	}
	if !isAlphaNumeric(rune(vendor[len(vendor)-1])) {
		return fmt.Errorf("invalid vendor %q, should end with a letter or digit", vendor)
	}

	return nil
}


// validateClassName checks the validity of class name.
// A class name may contain the following ASCII characters:
//   - upper- and lowercase letters ('A'-'Z', 'a'-'z')
//   - digits ('0'-'9')
//   - underscore and dash ('_', '-')
func validateClassName(class string) error {
	if class == "" {
		return fmt.Errorf("invalid (empty) device class")
	}
	if !isLetter(rune(class[0])) {
		return fmt.Errorf("invalid class %q, should start with letter", class)
	}
	for _, c := range string(class[1 : len(class)-1]) {
		switch {
		case isAlphaNumeric(c):
		case c == '_' || c == '-':
		default:
			return fmt.Errorf("invalid character '%c' in device class %q",
				c, class)
		}
	}
	if !isAlphaNumeric(rune(class[len(class)-1])) {
		return fmt.Errorf("invalid class %q, should end with a letter or digit", class)
	}
	return nil
}


// validateDeviceName checks the validity of a device name.
// A device name may contain the following ASCII characters:
//   - upper- and lowercase letters ('A'-'Z', 'a'-'z')
//   - digits ('0'-'9')
//   - underscore, dash, dot, colon ('_', '-', '.', ':')
func validateDeviceName(name string) error {
	if name == "" {
		return fmt.Errorf("invalid (empty) device name")
	}
	if !isAlphaNumeric(rune(name[0])) {
		return fmt.Errorf("invalid class %q, should start with a letter or digit", name)
	}
	if len(name) == 1 {
		return nil
	}
	for _, c := range string(name[1 : len(name)-1]) {
		switch {
		case isAlphaNumeric(c):
		case c == '_' || c == '-' || c == '.' || c == ':':
		default:
			return fmt.Errorf("invalid character '%c' in device name %q",
				c, name)
		}
	}
	if !isAlphaNumeric(rune(name[len(name)-1])) {
		return fmt.Errorf("invalid name %q, should end with a letter or digit", name)
	}
	return nil
}


func isLetter(c rune) bool

func isLetter(c rune) bool {
	return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z')
}


func isDigit(c rune) bool

func isDigit(c rune) bool {
	return '0' <= c && c <= '9'
}


func isAlphaNumeric(c rune) bool

func isAlphaNumeric(c rune) bool {
	return isLetter(c) || isDigit(c)
}


func (info *ClaimInfo) addPodReference(podUID types.UID) {
	info.Lock()
	defer info.Unlock()

	info.PodUIDs.Insert(string(podUID))
}


func (info *ClaimInfo) deletePodReference(podUID types.UID) {
	info.Lock()
	defer info.Unlock()

	info.PodUIDs.Delete(string(podUID))
}


func (info *ClaimInfo) addCDIDevices(pluginName string, cdiDevices []string) error {
	info.Lock()
	defer info.Unlock()

	// NOTE: Passing CDI device names as annotations is a temporary solution
	// It will be removed after all runtimes are updated
	// to get CDI device names from the ContainerConfig.CDIDevices field
	annotations, err := generateCDIAnnotations(info.ClaimUID, info.DriverName, cdiDevices)
	if err != nil {
		return fmt.Errorf("failed to generate container annotations, err: %+v", err)
	}

	if info.CDIDevices == nil {
		info.CDIDevices = make(map[string][]string)
	}

	info.CDIDevices[pluginName] = cdiDevices
	info.annotations = append(info.annotations, annotations...)

	return nil
}


func newClaimInfo(driverName, className string, claimUID types.UID, claimName, namespace string, podUIDs sets.Set[string]) *ClaimInfo {
	claimInfoState := state.ClaimInfoState{
		DriverName: driverName,
		ClassName:  className,
		ClaimUID:   claimUID,
		ClaimName:  claimName,
		Namespace:  namespace,
		PodUIDs:    podUIDs,
	}
	claimInfo := ClaimInfo{
		ClaimInfoState: claimInfoState,
	}
	return &claimInfo
}


// newClaimInfoCache is a function that returns an instance of the claimInfoCache.
func newClaimInfoCache(stateDir, checkpointName string) (*claimInfoCache, error) {
	stateImpl, err := state.NewCheckpointState(stateDir, checkpointName)
	if err != nil {
		return nil, fmt.Errorf("could not initialize checkpoint manager, please drain node and remove dra state file, err: %+v", err)
	}

	curState, err := stateImpl.GetOrCreate()
	if err != nil {
		return nil, fmt.Errorf("error calling GetOrCreate() on checkpoint state: %v", err)
	}

	cache := &claimInfoCache{
		state:     stateImpl,
		claimInfo: make(map[string]*ClaimInfo),
	}

	for _, entry := range curState {
		info := newClaimInfo(
			entry.DriverName,
			entry.ClassName,
			entry.ClaimUID,
			entry.ClaimName,
			entry.Namespace,
			entry.PodUIDs,
		)
		for pluginName, cdiDevices := range entry.CDIDevices {
			err := info.addCDIDevices(pluginName, cdiDevices)
			if err != nil {
				return nil, fmt.Errorf("failed to add CDIDevices to claimInfo %+v: %+v", info, err)
			}
		}
		cache.add(info)
	}

	return cache, nil
}


func (cache *claimInfoCache) add(res *ClaimInfo) {
	cache.Lock()
	defer cache.Unlock()

	cache.claimInfo[res.ClaimName+res.Namespace] = res
}


func (cache *claimInfoCache) get(claimName, namespace string) *ClaimInfo {
	cache.RLock()
	defer cache.RUnlock()

	return cache.claimInfo[claimName+namespace]
}


func (cache *claimInfoCache) delete(claimName, namespace string) {
	cache.Lock()
	defer cache.Unlock()

	delete(cache.claimInfo, claimName+namespace)
}


// hasPodReference checks if there is at least one claim
// that is referenced by the pod with the given UID
// This function is used indirectly by the status manager
// to check if pod can enter termination status
func (cache *claimInfoCache) hasPodReference(UID types.UID) bool {
	cache.RLock()
	defer cache.RUnlock()

	for _, claimInfo := range cache.claimInfo {
		if claimInfo.PodUIDs.Has(string(UID)) {
			return true
		}
	}

	return false
}


func (cache *claimInfoCache) syncToCheckpoint() error {
	cache.RLock()
	defer cache.RUnlock()

	claimInfoStateList := make(state.ClaimInfoStateList, 0, len(cache.claimInfo))
	for _, infoClaim := range cache.claimInfo {
		claimInfoStateList = append(claimInfoStateList, infoClaim.ClaimInfoState)
	}

	return cache.state.Store(claimInfoStateList)
}


// NewManagerImpl creates a new manager.
func NewManagerImpl(kubeClient clientset.Interface, stateFileDirectory string) (*ManagerImpl, error) {
	klog.V(2).InfoS("Creating DRA manager")

	claimInfoCache, err := newClaimInfoCache(stateFileDirectory, draManagerStateFileName)
	if err != nil {
		return nil, fmt.Errorf("failed to create claimInfo cache: %+v", err)
	}

	manager := &ManagerImpl{
		cache:      claimInfoCache,
		kubeClient: kubeClient,
	}

	return manager, nil
}


// PrepareResources attempts to prepare all of the required resource
// plugin resources for the input container, issue an NodePrepareResource rpc request
// for each new resource requirement, process their responses and update the cached
// containerResources on success.
func (m *ManagerImpl) PrepareResources(pod *v1.Pod) error {
	// Process resources for each resource claim referenced by container
	for _, container := range append(pod.Spec.InitContainers, pod.Spec.Containers...) {
		for range container.Resources.Claims {
			for i := range pod.Spec.ResourceClaims {
				claimName := resourceclaim.Name(pod, &pod.Spec.ResourceClaims[i])
				klog.V(3).InfoS("Processing resource", "claim", claimName, "pod", pod.Name)

				// Resource is already prepared, add pod UID to it
				if claimInfo := m.cache.get(claimName, pod.Namespace); claimInfo != nil {
					// We delay checkpointing of this change until this call
					// returns successfully. It is OK to do this because we
					// will only return successfully from this call if the
					// checkpoint has succeeded. That means if the kubelet is
					// ever restarted before this checkpoint succeeds, the pod
					// whose resources are being prepared would never have
					// started, so it's OK (actually correct) to not include it
					// in the cache.
					claimInfo.addPodReference(pod.UID)
					continue
				}

				// Query claim object from the API server
				resourceClaim, err := m.kubeClient.ResourceV1alpha2().ResourceClaims(pod.Namespace).Get(
					context.TODO(),
					claimName,
					metav1.GetOptions{})
				if err != nil {
					return fmt.Errorf("failed to fetch ResourceClaim %s referenced by pod %s: %+v", claimName, pod.Name, err)
				}

				// Check if pod is in the ReservedFor for the claim
				if !resourceclaim.IsReservedForPod(pod, resourceClaim) {
					return fmt.Errorf("pod %s(%s) is not allowed to use resource claim %s(%s)",
						pod.Name, pod.UID, claimName, resourceClaim.UID)
				}

				// Grab the allocation.resourceHandles. If there are no
				// allocation.resourceHandles, create a single resourceHandle with no
				// content. This will trigger processing of this claim by a single
				// kubelet plugin whose name matches resourceClaim.Status.DriverName.
				resourceHandles := resourceClaim.Status.Allocation.ResourceHandles
				if len(resourceHandles) == 0 {
					resourceHandles = make([]resourcev1alpha2.ResourceHandle, 1)
				}

				// Create a claimInfo object to store the relevant claim info.
				claimInfo := newClaimInfo(
					resourceClaim.Status.DriverName,
					resourceClaim.Spec.ResourceClassName,
					resourceClaim.UID,
					resourceClaim.Name,
					resourceClaim.Namespace,
					sets.New(string(pod.UID)),
				)

				// Walk through each resourceHandle
				for _, resourceHandle := range resourceHandles {
					// If no DriverName is provided in the resourceHandle, we
					// use the DriverName from the status
					pluginName := resourceHandle.DriverName
					if pluginName == "" {
						pluginName = resourceClaim.Status.DriverName
					}

					// Call NodePrepareResource RPC for each resourceHandle
					client, err := dra.NewDRAPluginClient(pluginName)
					if err != nil {
						return fmt.Errorf("failed to get DRA Plugin client for plugin name %s, err=%+v", pluginName, err)
					}
					response, err := client.NodePrepareResource(
						context.Background(),
						resourceClaim.Namespace,
						resourceClaim.UID,
						resourceClaim.Name,
						resourceHandle.Data)
					if err != nil {
						return fmt.Errorf("NodePrepareResource failed, claim UID: %s, claim name: %s, resource handle: %s, err: %+v",
							resourceClaim.UID, resourceClaim.Name, resourceHandle.Data, err)
					}
					klog.V(3).InfoS("NodePrepareResource succeeded", "pluginName", pluginName, "response", response)

					// Add the CDI Devices returned by NodePrepareResource to
					// the claimInfo object.
					err = claimInfo.addCDIDevices(pluginName, response.CdiDevices)
					if err != nil {
						return fmt.Errorf("failed to add CDIDevices to claimInfo %+v: %+v", claimInfo, err)
					}

					// TODO: We (re)add the claimInfo object to the cache and
					// sync it to the checkpoint *after* the
					// NodePrepareResource call has completed. This will cause
					// issues if the kubelet gets restarted between
					// NodePrepareResource and syncToCheckpoint. It will result
					// in not calling NodeUnprepareResource for this claim
					// because no claimInfo will be synced back to the cache
					// for it after the restart. We need to resolve this issue
					// before moving to beta.
					m.cache.add(claimInfo)

					// Checkpoint to reduce redundant calls to
					// NodePrepareResource() after a kubelet restart.
					err = m.cache.syncToCheckpoint()
					if err != nil {
						return fmt.Errorf("failed to checkpoint claimInfo state, err: %+v", err)
					}
				}
			}
		}
	}
	// Checkpoint to capture all of the previous addPodReference() calls.
	err := m.cache.syncToCheckpoint()
	if err != nil {
		return fmt.Errorf("failed to checkpoint claimInfo state, err: %+v", err)
	}
	return nil
}


// GetResources gets a ContainerInfo object from the claimInfo cache.
// This information is used by the caller to update a container config.
func (m *ManagerImpl) GetResources(pod *v1.Pod, container *v1.Container) (*ContainerInfo, error) {
	annotations := []kubecontainer.Annotation{}
	cdiDevices := []kubecontainer.CDIDevice{}

	for i, podResourceClaim := range pod.Spec.ResourceClaims {
		claimName := resourceclaim.Name(pod, &pod.Spec.ResourceClaims[i])

		for _, claim := range container.Resources.Claims {
			if podResourceClaim.Name != claim.Name {
				continue
			}

			claimInfo := m.cache.get(claimName, pod.Namespace)
			if claimInfo == nil {
				return nil, fmt.Errorf("unable to get resource for namespace: %s, claim: %s", pod.Namespace, claimName)
			}

			klog.V(3).InfoS("Add resource annotations", "claim", claimName, "annotations", claimInfo.annotations)
			annotations = append(annotations, claimInfo.annotations...)
			for _, devices := range claimInfo.CDIDevices {
				for _, device := range devices {
					cdiDevices = append(cdiDevices, kubecontainer.CDIDevice{Name: device})
				}
			}
		}
	}

	return &ContainerInfo{Annotations: annotations, CDIDevices: cdiDevices}, nil
}


// UnprepareResources calls a plugin's NodeUnprepareResource API for each resource claim owned by a pod.
// This function is idempotent and may be called multiple times against the same pod.
// As such, calls to the underlying NodeUnprepareResource API are skipped for claims that have
// already been successfully unprepared.
func (m *ManagerImpl) UnprepareResources(pod *v1.Pod) error {
	// Call NodeUnprepareResource RPC for every resource claim referenced by the pod
	for i := range pod.Spec.ResourceClaims {
		claimName := resourceclaim.Name(pod, &pod.Spec.ResourceClaims[i])
		claimInfo := m.cache.get(claimName, pod.Namespace)

		// Skip calling NodeUnprepareResource if claim info is not cached
		if claimInfo == nil {
			continue
		}

		// Skip calling NodeUnprepareResource if other pods are still referencing it
		if len(claimInfo.PodUIDs) > 1 {
			// We delay checkpointing of this change until this call returns successfully.
			// It is OK to do this because we will only return successfully from this call if
			// the checkpoint has succeeded. That means if the kubelet is ever restarted
			// before this checkpoint succeeds, we will simply call into this (idempotent)
			// function again.
			claimInfo.deletePodReference(pod.UID)
			continue
		}

		// Query claim object from the API server
		resourceClaim, err := m.kubeClient.ResourceV1alpha2().ResourceClaims(pod.Namespace).Get(
			context.TODO(),
			claimName,
			metav1.GetOptions{})
		if err != nil {
			return fmt.Errorf("failed to fetch ResourceClaim %s referenced by pod %s: %+v", claimName, pod.Name, err)
		}

		// Grab the allocation.resourceHandles. If there are no
		// allocation.resourceHandles, create a single resourceHandle with no
		// content. This will trigger processing of this claim by a single
		// kubelet plugin whose name matches resourceClaim.Status.DriverName.
		resourceHandles := resourceClaim.Status.Allocation.ResourceHandles
		if len(resourceHandles) == 0 {
			resourceHandles = make([]resourcev1alpha2.ResourceHandle, 1)
		}

		// Loop through all plugins and call NodeUnprepareResource only for the
		// last pod that references the claim
		for _, resourceHandle := range resourceHandles {
			// If no DriverName is provided in the resourceHandle, we
			// use the DriverName from the status
			pluginName := resourceHandle.DriverName
			if pluginName == "" {
				pluginName = claimInfo.DriverName
			}

			// Call NodeUnprepareResource RPC for each resourceHandle
			client, err := dra.NewDRAPluginClient(pluginName)
			if err != nil {
				return fmt.Errorf("failed to get DRA Plugin client for plugin name %s, err=%+v", pluginName, err)
			}
			response, err := client.NodeUnprepareResource(
				context.Background(),
				claimInfo.Namespace,
				claimInfo.ClaimUID,
				claimInfo.ClaimName,
				resourceHandle.Data)
			if err != nil {
				return fmt.Errorf(
					"NodeUnprepareResource failed, pod: %s, claim UID: %s, claim name: %s, CDI devices: %s, err: %+v",
					pod.Name, claimInfo.ClaimUID, claimInfo.ClaimName, claimInfo.CDIDevices, err)
			}
			klog.V(3).InfoS("NodeUnprepareResource succeeded", "response", response)
		}

		// Delete last pod UID only if all NodeUnprepareResource calls succeed.
		// This ensures that the status manager doesn't enter termination status
		// for the pod. This logic is implemented in
		// m.PodMightNeedToUnprepareResources and claimInfo.hasPodReference.
		claimInfo.deletePodReference(pod.UID)
		m.cache.delete(claimInfo.ClaimName, pod.Namespace)

		// Checkpoint to reduce redundant calls to NodeUnPrepareResource() after a kubelet restart.
		err = m.cache.syncToCheckpoint()
		if err != nil {
			return fmt.Errorf("failed to checkpoint claimInfo state, err: %+v", err)
		}
	}
	// Checkpoint to capture all of the previous deletePodReference() calls.
	err := m.cache.syncToCheckpoint()
	if err != nil {
		return fmt.Errorf("failed to checkpoint claimInfo state, err: %+v", err)
	}
	return nil
}


// PodMightNeedToUnprepareResources returns true if the pod might need to
// unprepare resources
func (m *ManagerImpl) PodMightNeedToUnprepareResources(UID types.UID) bool {
	return m.cache.hasPodReference(UID)
}


// GetCongtainerClaimInfos gets Container's ClaimInfo
func (m *ManagerImpl) GetContainerClaimInfos(pod *v1.Pod, container *v1.Container) ([]*ClaimInfo, error) {
	claimInfos := make([]*ClaimInfo, 0, len(pod.Spec.ResourceClaims))

	for i, podResourceClaim := range pod.Spec.ResourceClaims {
		claimName := resourceclaim.Name(pod, &pod.Spec.ResourceClaims[i])

		for _, claim := range container.Resources.Claims {
			if podResourceClaim.Name != claim.Name {
				continue
			}
			claimInfo := m.cache.get(claimName, pod.Namespace)
			if claimInfo == nil {
				return nil, fmt.Errorf("unable to get resource for namespace: %s, claim: %s", pod.Namespace, claimName)
			}
			claimInfos = append(claimInfos, claimInfo)
		}
	}
	return claimInfos, nil
}


// newV1NodeClient creates a new NodeClient with the internally used gRPC
// connection set up. It also returns a closer which must be called to close
// the gRPC connection when the NodeClient is not used anymore.
// This is the default implementation for the nodeV1ClientCreator, used in
// newDRAPluginClient.
func newV1NodeClient(addr draAddr) (nodeClient drapbv1.NodeClient, closer io.Closer, err error) {
	var conn *grpc.ClientConn

	conn, err = newGrpcConn(addr)
	if err != nil {
		return nil, nil, err
	}

	return drapbv1.NewNodeClient(conn), conn, nil
}


func NewDRAPluginClient(pluginName string) (Client, error) {
	if pluginName == "" {
		return nil, fmt.Errorf("plugin name is empty")
	}

	existingPlugin := draPlugins.Get(pluginName)
	if existingPlugin == nil {
		return nil, fmt.Errorf("plugin name %s not found in the list of registered DRA plugins", pluginName)
	}

	return &draPluginClient{
		pluginName:          pluginName,
		addr:                draAddr(existingPlugin.endpoint),
		nodeV1ClientCreator: newV1NodeClient,
	}, nil
}


func (r *draPluginClient) NodePrepareResource(
	ctx context.Context,
	namespace string,
	claimUID types.UID,
	claimName string,
	resourceHandle string,
) (*drapbv1.NodePrepareResourceResponse, error) {
	klog.V(4).InfoS(
		log("calling NodePrepareResource rpc"),
		"namespace", namespace,
		"claimUID", claimUID,
		"claimName", claimName,
		"resourceHandle", resourceHandle)

	if r.nodeV1ClientCreator == nil {
		return nil, errors.New("failed to call NodePrepareResource. nodeV1ClientCreator is nil")
	}

	nodeClient, closer, err := r.nodeV1ClientCreator(r.addr)
	if err != nil {
		return nil, err
	}
	defer closer.Close()

	req := &drapbv1.NodePrepareResourceRequest{
		Namespace:      namespace,
		ClaimUid:       string(claimUID),
		ClaimName:      claimName,
		ResourceHandle: resourceHandle,
	}

	ctx, cancel := context.WithTimeout(ctx, PluginClientTimeout)
	defer cancel()

	return nodeClient.NodePrepareResource(ctx, req)
}


func (r *draPluginClient) NodeUnprepareResource(
	ctx context.Context,
	namespace string,
	claimUID types.UID,
	claimName string,
	resourceHandle string,
) (*drapbv1.NodeUnprepareResourceResponse, error) {
	klog.V(4).InfoS(
		log("calling NodeUnprepareResource rpc"),
		"namespace", namespace,
		"claimUID", claimUID,
		"claimname", claimName,
		"resourceHandle", resourceHandle)

	if r.nodeV1ClientCreator == nil {
		return nil, errors.New("nodeV1ClientCreate is nil")
	}

	nodeClient, closer, err := r.nodeV1ClientCreator(r.addr)
	if err != nil {
		return nil, err
	}
	defer closer.Close()

	req := &drapbv1.NodeUnprepareResourceRequest{
		Namespace:      namespace,
		ClaimUid:       string(claimUID),
		ClaimName:      claimName,
		ResourceHandle: resourceHandle,
	}

	ctx, cancel := context.WithTimeout(ctx, PluginClientTimeout)
	defer cancel()

	return nodeClient.NodeUnprepareResource(ctx, req)
}


func newGrpcConn(addr draAddr) (*grpc.ClientConn, error) {
	network := "unix"
	klog.V(4).InfoS(log("creating new gRPC connection"), "protocol", network, "endpoint", addr)

	return grpc.Dial(
		string(addr),
		grpc.WithTransportCredentials(insecure.NewCredentials()),
		grpc.WithContextDialer(func(ctx context.Context, target string) (net.Conn, error) {
			return (&net.Dialer{}).DialContext(ctx, network, target)
		}),
	)
}


// NewPluginHandler returns new registration handler.
func NewRegistrationHandler() *RegistrationHandler {
	return &RegistrationHandler{}
}


// RegisterPlugin is called when a plugin can be registered.
func (h *RegistrationHandler) RegisterPlugin(pluginName string, endpoint string, versions []string) error {
	klog.InfoS("Register new DRA plugin", "name", pluginName, "endpoint", endpoint)

	highestSupportedVersion, err := h.validateVersions("RegisterPlugin", pluginName, versions)
	if err != nil {
		return err
	}

	// Storing endpoint of newly registered DRA Plugin into the map, where plugin name will be the key
	// all other DRA components will be able to get the actual socket of DRA plugins by its name.
	draPlugins.Set(pluginName, &Plugin{
		endpoint:                endpoint,
		highestSupportedVersion: highestSupportedVersion,
	})

	return nil
}


// Return the highest supported version.
func highestSupportedVersion(versions []string) (*utilversion.Version, error) {
	if len(versions) == 0 {
		return nil, errors.New(log("DRA plugin reporting empty array for supported versions"))
	}

	var highestSupportedVersion *utilversion.Version

	var theErr error

	for i := len(versions) - 1; i >= 0; i-- {
		currentHighestVer, err := utilversion.ParseGeneric(versions[i])
		if err != nil {
			theErr = err

			continue
		}

		if currentHighestVer.Major() > 1 {
			// DRA currently only has version 1.x
			continue
		}

		if highestSupportedVersion == nil || highestSupportedVersion.LessThan(currentHighestVer) {
			highestSupportedVersion = currentHighestVer
		}
	}

	if highestSupportedVersion == nil {
		return nil, fmt.Errorf(
			"could not find a highest supported version from versions (%v) reported by this plugin: %+v",
			versions, theErr)
	}

	if highestSupportedVersion.Major() != 1 {
		return nil, fmt.Errorf("highest supported version reported by plugin is %v, must be v1.x", highestSupportedVersion)
	}

	return highestSupportedVersion, nil
}


func (h *RegistrationHandler) validateVersions(
	callerName string,
	pluginName string,
	versions []string,
) (*utilversion.Version, error) {
	if len(versions) == 0 {
		return nil, errors.New(
			log(
				"%s for DRA plugin %q failed. Plugin returned an empty list for supported versions",
				callerName,
				pluginName,
			),
		)
	}

	// Validate version
	newPluginHighestVersion, err := highestSupportedVersion(versions)
	if err != nil {
		return nil, errors.New(
			log(
				"%s for DRA plugin %q failed. None of the versions specified %q are supported. err=%v",
				callerName,
				pluginName,
				versions,
				err,
			),
		)
	}

	existingPlugin := draPlugins.Get(pluginName)
	if existingPlugin != nil {
		if !existingPlugin.highestSupportedVersion.LessThan(newPluginHighestVersion) {
			return nil, errors.New(
				log(
					"%s for DRA plugin %q failed. Another plugin with the same name is already registered with a higher supported version: %q",
					callerName,
					pluginName,
					existingPlugin.highestSupportedVersion,
				),
			)
		}
	}

	return newPluginHighestVersion, nil
}


func unregisterPlugin(pluginName string) {
	draPlugins.Delete(pluginName)
}


// DeRegisterPlugin is called when a plugin has removed its socket,
// signaling it is no longer available.
func (h *RegistrationHandler) DeRegisterPlugin(pluginName string) {
	klog.InfoS("DeRegister DRA plugin", "name", pluginName)
	unregisterPlugin(pluginName)
}


// ValidatePlugin is called by kubelet's plugin watcher upon detection
// of a new registration socket opened by DRA plugin.
func (h *RegistrationHandler) ValidatePlugin(pluginName string, endpoint string, versions []string) error {
	klog.InfoS("Validate DRA plugin", "name", pluginName, "endpoint", endpoint, "versions", strings.Join(versions, ","))

	_, err := h.validateVersions("ValidatePlugin", pluginName, versions)
	if err != nil {
		return fmt.Errorf("validation failed for DRA plugin %s at endpoint %s: %+v", pluginName, endpoint, err)
	}

	return err
}


// log prepends log string with `kubernetes.io/dra`.
func log(msg string, parts ...interface{}) string {
	return fmt.Sprintf(fmt.Sprintf("%s: %s", DRAPluginName, msg), parts...)
}


// Get lets you retrieve a DRA Plugin by name.
// This method is protected by a mutex.
func (s *PluginsStore) Get(pluginName string) *Plugin {
	s.RLock()
	defer s.RUnlock()

	return s.store[pluginName]
}


// Set lets you save a DRA Plugin to the list and give it a specific name.
// This method is protected by a mutex.
func (s *PluginsStore) Set(pluginName string, plugin *Plugin) {
	s.Lock()
	defer s.Unlock()

	if s.store == nil {
		s.store = make(map[string]*Plugin)
	}

	s.store[pluginName] = plugin
}


// Delete lets you delete a DRA Plugin by name.
// This method is protected by a mutex.
func (s *PluginsStore) Delete(pluginName string) {
	s.Lock()
	defer s.Unlock()

	delete(s.store, pluginName)
}


func (s *PluginsStore) Clear()

// Clear deletes all entries in the store.
// This methiod is protected by a mutex.
func (s *PluginsStore) Clear() {
	s.Lock()
	defer s.Unlock()

	s.store = make(map[string]*Plugin)
}


// NewDRAManagerCheckpoint returns an instance of Checkpoint
func NewDRAManagerCheckpoint() *DRAManagerCheckpoint {
	return &DRAManagerCheckpoint{
		Version: checkpointVersion,
		Entries: ClaimInfoStateList{},
	}
}


// MarshalCheckpoint returns marshalled checkpoint
func (dc *DRAManagerCheckpoint) MarshalCheckpoint() ([]byte, error) {
	// make sure checksum wasn't set before so it doesn't affect output checksum
	dc.Checksum = 0
	dc.Checksum = checksum.New(dc)
	return json.Marshal(*dc)
}


// UnmarshalCheckpoint tries to unmarshal passed bytes to checkpoint
func (dc *DRAManagerCheckpoint) UnmarshalCheckpoint(blob []byte) error {
	return json.Unmarshal(blob, dc)
}


// VerifyChecksum verifies that current checksum of checkpoint is valid
func (dc *DRAManagerCheckpoint) VerifyChecksum() error {
	ck := dc.Checksum
	dc.Checksum = 0
	err := ck.Verify(dc)
	dc.Checksum = ck
	return err
}


// NewCheckpointState creates new State for keeping track of claim info  with checkpoint backend
func NewCheckpointState(stateDir, checkpointName string) (*stateCheckpoint, error) {
	checkpointManager, err := checkpointmanager.NewCheckpointManager(stateDir)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize checkpoint manager: %v", err)
	}
	stateCheckpoint := &stateCheckpoint{
		checkpointManager: checkpointManager,
		checkpointName:    checkpointName,
	}

	return stateCheckpoint, nil
}


// get state from a checkpoint and creates it if it doesn't exist
func (sc *stateCheckpoint) GetOrCreate() (ClaimInfoStateList, error) {
	sc.Lock()
	defer sc.Unlock()

	checkpoint := NewDRAManagerCheckpoint()
	err := sc.checkpointManager.GetCheckpoint(sc.checkpointName, checkpoint)
	if err == errors.ErrCheckpointNotFound {
		sc.store(ClaimInfoStateList{})
		return ClaimInfoStateList{}, nil
	}
	if err != nil {
		return nil, fmt.Errorf("failed to get checkpoint %v: %v", sc.checkpointName, err)
	}

	return checkpoint.Entries, nil
}


// saves state to a checkpoint
func (sc *stateCheckpoint) Store(claimInfoStateList ClaimInfoStateList) error {
	sc.Lock()
	defer sc.Unlock()

	return sc.store(claimInfoStateList)
}


// saves state to a checkpoint, caller is responsible for locking
func (sc *stateCheckpoint) store(claimInfoStateList ClaimInfoStateList) error {
	checkpoint := NewDRAManagerCheckpoint()
	checkpoint.Entries = claimInfoStateList

	err := sc.checkpointManager.CreateCheckpoint(sc.checkpointName, checkpoint)
	if err != nil {
		return fmt.Errorf("could not save checkpoint %s: %v", sc.checkpointName, err)
	}
	return nil
}


// hardEvictionReservation returns a resourcelist that includes reservation of resources based on hard eviction thresholds.
func hardEvictionReservation(thresholds []evictionapi.Threshold, capacity v1.ResourceList) v1.ResourceList {
	if len(thresholds) == 0 {
		return nil
	}
	ret := v1.ResourceList{}
	for _, threshold := range thresholds {
		if threshold.Operator != evictionapi.OpLessThan {
			continue
		}
		switch threshold.Signal {
		case evictionapi.SignalMemoryAvailable:
			memoryCapacity := capacity[v1.ResourceMemory]
			value := evictionapi.GetThresholdQuantity(threshold.Value, &memoryCapacity)
			ret[v1.ResourceMemory] = *value
		case evictionapi.SignalNodeFsAvailable:
			storageCapacity := capacity[v1.ResourceEphemeralStorage]
			value := evictionapi.GetThresholdQuantity(threshold.Value, &storageCapacity)
			ret[v1.ResourceEphemeralStorage] = *value
		}
	}
	return ret
}


// MilliCPUToQuota converts milliCPU to CFS quota and period values.
// Input parameters and resulting value is number of microseconds.
func MilliCPUToQuota(milliCPU int64, period int64) (quota int64) {
	// CFS quota is measured in two values:
	//  - cfs_period_us=100ms (the amount of time to measure usage across given by period)
	//  - cfs_quota=20ms (the amount of cpu time allowed to be used across a period)
	// so in the above example, you are limited to 20% of a single CPU
	// for multi-cpu environments, you just scale equivalent amounts
	// see https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt for details

	if milliCPU == 0 {
		return
	}

	if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CPUCFSQuotaPeriod) {
		period = QuotaPeriod
	}

	// we then convert your milliCPU to a value normalized over a period
	quota = (milliCPU * period) / MilliCPUToCPU

	// quota needs to be a minimum of 1ms.
	if quota < MinQuotaPeriod {
		quota = MinQuotaPeriod
	}
	return
}


// MilliCPUToShares converts the milliCPU to CFS shares.
func MilliCPUToShares(milliCPU int64) uint64 {
	if milliCPU == 0 {
		// Docker converts zero milliCPU to unset, which maps to kernel default
		// for unset: 1024. Return 2 here to really match kernel default for
		// zero milliCPU.
		return MinShares
	}
	// Conceptually (milliCPU / milliCPUToCPU) * sharesPerCPU, but factored to improve rounding.
	shares := (milliCPU * SharesPerCPU) / MilliCPUToCPU
	if shares < MinShares {
		return MinShares
	}
	if shares > MaxShares {
		return MaxShares
	}
	return uint64(shares)
}


// HugePageLimits converts the API representation to a map
// from huge page size (in bytes) to huge page limit (in bytes).
func HugePageLimits(resourceList v1.ResourceList) map[int64]int64 {
	hugePageLimits := map[int64]int64{}
	for k, v := range resourceList {
		if v1helper.IsHugePageResourceName(k) {
			pageSize, _ := v1helper.HugePageSizeFromResourceName(k)
			if value, exists := hugePageLimits[pageSize.Value()]; exists {
				hugePageLimits[pageSize.Value()] = value + v.Value()
			} else {
				hugePageLimits[pageSize.Value()] = v.Value()
			}
		}
	}
	return hugePageLimits
}


// ResourceConfigForPod takes the input pod and outputs the cgroup resource config.
func ResourceConfigForPod(pod *v1.Pod, enforceCPULimits bool, cpuPeriod uint64, enforceMemoryQoS bool) *ResourceConfig {
	inPlacePodVerticalScalingEnabled := utilfeature.DefaultFeatureGate.Enabled(kubefeatures.InPlacePodVerticalScaling)
	// sum requests and limits.
	reqs := resource.PodRequests(pod, resource.PodResourcesOptions{
		InPlacePodVerticalScalingEnabled: inPlacePodVerticalScalingEnabled,
	})
	// track if limits were applied for each resource.
	memoryLimitsDeclared := true
	cpuLimitsDeclared := true

	limits := resource.PodLimits(pod, resource.PodResourcesOptions{
		InPlacePodVerticalScalingEnabled: inPlacePodVerticalScalingEnabled,
		ContainerFn: func(res v1.ResourceList, containerType podutil.ContainerType) {
			if res.Cpu().IsZero() {
				cpuLimitsDeclared = false
			}
			if res.Memory().IsZero() {
				memoryLimitsDeclared = false
			}
		},
	})
	// map hugepage pagesize (bytes) to limits (bytes)
	hugePageLimits := HugePageLimits(reqs)

	cpuRequests := int64(0)
	cpuLimits := int64(0)
	memoryLimits := int64(0)
	if request, found := reqs[v1.ResourceCPU]; found {
		cpuRequests = request.MilliValue()
	}
	if limit, found := limits[v1.ResourceCPU]; found {
		cpuLimits = limit.MilliValue()
	}
	if limit, found := limits[v1.ResourceMemory]; found {
		memoryLimits = limit.Value()
	}

	// convert to CFS values
	cpuShares := MilliCPUToShares(cpuRequests)
	cpuQuota := MilliCPUToQuota(cpuLimits, int64(cpuPeriod))

	// quota is not capped when cfs quota is disabled
	if !enforceCPULimits {
		cpuQuota = int64(-1)
	}

	// determine the qos class
	qosClass := v1qos.GetPodQOS(pod)

	// build the result
	result := &ResourceConfig{}
	if qosClass == v1.PodQOSGuaranteed {
		result.CPUShares = &cpuShares
		result.CPUQuota = &cpuQuota
		result.CPUPeriod = &cpuPeriod
		result.Memory = &memoryLimits
	} else if qosClass == v1.PodQOSBurstable {
		result.CPUShares = &cpuShares
		if cpuLimitsDeclared {
			result.CPUQuota = &cpuQuota
			result.CPUPeriod = &cpuPeriod
		}
		if memoryLimitsDeclared {
			result.Memory = &memoryLimits
		}
	} else {
		shares := uint64(MinShares)
		result.CPUShares = &shares
	}
	result.HugePageLimit = hugePageLimits

	if enforceMemoryQoS {
		memoryMin := int64(0)
		if request, found := reqs[v1.ResourceMemory]; found {
			memoryMin = request.Value()
		}
		if memoryMin > 0 {
			result.Unified = map[string]string{
				MemoryMin: strconv.FormatInt(memoryMin, 10),
			}
		}
	}

	return result
}


// getCgroupSubsystemsV1 returns information about the mounted cgroup v1 subsystems
func getCgroupSubsystemsV1() (*CgroupSubsystems, error) {
	// get all cgroup mounts.
	allCgroups, err := libcontainercgroups.GetCgroupMounts(true)
	if err != nil {
		return &CgroupSubsystems{}, err
	}
	if len(allCgroups) == 0 {
		return &CgroupSubsystems{}, fmt.Errorf("failed to find cgroup mounts")
	}
	mountPoints := make(map[string]string, len(allCgroups))
	for _, mount := range allCgroups {
		// BEFORE kubelet used a random mount point per cgroups subsystem;
		// NOW    more deterministic: kubelet use mount point with shortest path;
		// FUTURE is bright with clear expectation determined in doc.
		// ref. issue: https://github.com/kubernetes/kubernetes/issues/95488

		for _, subsystem := range mount.Subsystems {
			previous := mountPoints[subsystem]
			if previous == "" || len(mount.Mountpoint) < len(previous) {
				mountPoints[subsystem] = mount.Mountpoint
			}
		}
	}
	return &CgroupSubsystems{
		Mounts:      allCgroups,
		MountPoints: mountPoints,
	}, nil
}


// getCgroupSubsystemsV2 returns information about the enabled cgroup v2 subsystems
func getCgroupSubsystemsV2() (*CgroupSubsystems, error) {
	controllers, err := libcontainercgroups.GetAllSubsystems()
	if err != nil {
		return nil, err
	}

	mounts := []libcontainercgroups.Mount{}
	mountPoints := make(map[string]string, len(controllers))
	for _, controller := range controllers {
		mountPoints[controller] = util.CgroupRoot
		m := libcontainercgroups.Mount{
			Mountpoint: util.CgroupRoot,
			Root:       util.CgroupRoot,
			Subsystems: []string{controller},
		}
		mounts = append(mounts, m)
	}

	return &CgroupSubsystems{
		Mounts:      mounts,
		MountPoints: mountPoints,
	}, nil
}


// GetCgroupSubsystems returns information about the mounted cgroup subsystems
func GetCgroupSubsystems() (*CgroupSubsystems, error) {
	if libcontainercgroups.IsCgroup2UnifiedMode() {
		return getCgroupSubsystemsV2()
	}

	return getCgroupSubsystemsV1()
}


// getCgroupProcs takes a cgroup directory name as an argument
// reads through the cgroup's procs file and returns a list of tgid's.
// It returns an empty list if a procs file doesn't exists
func getCgroupProcs(dir string) ([]int, error) {
	procsFile := filepath.Join(dir, "cgroup.procs")
	f, err := os.Open(procsFile)
	if err != nil {
		if os.IsNotExist(err) {
			// The procsFile does not exist, So no pids attached to this directory
			return []int{}, nil
		}
		return nil, err
	}
	defer f.Close()

	s := bufio.NewScanner(f)
	out := []int{}
	for s.Scan() {
		if t := s.Text(); t != "" {
			pid, err := strconv.Atoi(t)
			if err != nil {
				return nil, fmt.Errorf("unexpected line in %v; could not convert to pid: %v", procsFile, err)
			}
			out = append(out, pid)
		}
	}
	return out, nil
}


// GetPodCgroupNameSuffix returns the last element of the pod CgroupName identifier
func GetPodCgroupNameSuffix(podUID types.UID) string {
	return podCgroupNamePrefix + string(podUID)
}


// NodeAllocatableRoot returns the literal cgroup path for the node allocatable cgroup
func NodeAllocatableRoot(cgroupRoot string, cgroupsPerQOS bool, cgroupDriver string) string {
	nodeAllocatableRoot := ParseCgroupfsToCgroupName(cgroupRoot)
	if cgroupsPerQOS {
		nodeAllocatableRoot = NewCgroupName(nodeAllocatableRoot, defaultNodeAllocatableCgroupName)
	}
	if cgroupDriver == "systemd" {
		return nodeAllocatableRoot.ToSystemd()
	}
	return nodeAllocatableRoot.ToCgroupfs()
}


// GetKubeletContainer returns the cgroup the kubelet will use
func GetKubeletContainer(kubeletCgroups string) (string, error) {
	if kubeletCgroups == "" {
		cont, err := getContainer(os.Getpid())
		if err != nil {
			return "", err
		}
		return cont, nil
	}
	return kubeletCgroups, nil
}


// MilliCPUToQuota converts milliCPU and period to CFS quota values.
func MilliCPUToQuota(milliCPU, period int64) int64 {
	return 0
}


// MilliCPUToShares converts the milliCPU to CFS shares.
func MilliCPUToShares(milliCPU int64) int64 {
	return 0
}


// ResourceConfigForPod takes the input pod and outputs the cgroup resource config.
func ResourceConfigForPod(pod *v1.Pod, enforceCPULimit bool, cpuPeriod uint64, enforceMemoryQoS bool) *ResourceConfig {
	return nil
}


// GetCgroupSubsystems returns information about the mounted cgroup subsystems
func GetCgroupSubsystems() (*CgroupSubsystems, error) {
	return nil, nil
}


func getCgroupProcs(dir string) ([]int, error) {
	return nil, nil
}


// GetPodCgroupNameSuffix returns the last element of the pod CgroupName identifier
func GetPodCgroupNameSuffix(podUID types.UID) string {
	return ""
}


// NodeAllocatableRoot returns the literal cgroup path for the node allocatable cgroup
func NodeAllocatableRoot(cgroupRoot string, cgroupsPerQOS bool, cgroupDriver string) string {
	return ""
}


// GetKubeletContainer returns the cgroup the kubelet will use
func GetKubeletContainer(kubeletCgroups string) (string, error) {
	return "", nil
}


func (i *internalContainerLifecycleImpl) PreStartContainer(pod *v1.Pod, container *v1.Container, containerID string) error {
	if i.cpuManager != nil {
		i.cpuManager.AddContainer(pod, container, containerID)
	}

	if i.memoryManager != nil {
		i.memoryManager.AddContainer(pod, container, containerID)
	}

	i.topologyManager.AddContainer(pod, container, containerID)

	return nil
}


func (i *internalContainerLifecycleImpl) PostStopContainer(containerID string) error {
	return i.topologyManager.RemoveContainer(containerID)
}


func (i *internalContainerLifecycleImpl) PreCreateContainer(pod *v1.Pod, container *v1.Container, containerConfig *runtimeapi.ContainerConfig) error {
	if i.cpuManager != nil {
		allocatedCPUs := i.cpuManager.GetCPUAffinity(string(pod.UID), container.Name)
		if !allocatedCPUs.IsEmpty() {
			containerConfig.Linux.Resources.CpusetCpus = allocatedCPUs.String()
		}
	}

	if i.memoryManager != nil {
		numaNodes := i.memoryManager.GetMemoryNUMANodes(pod, container)
		if numaNodes.Len() > 0 {
			var affinity []string
			for _, numaNode := range numaNodes.List() {
				affinity = append(affinity, strconv.Itoa(numaNode))
			}
			containerConfig.Linux.Resources.CpusetMems = strings.Join(affinity, ",")
		}
	}

	return nil
}


func (i *internalContainerLifecycleImpl) PreCreateContainer(pod *v1.Pod, container *v1.Container, containerConfig *runtimeapi.ContainerConfig) error {
	return nil
}


func (s *sourcesReadyStub) AddSource(source string) {}


func (s *sourcesReadyStub) AllReady() bool          { return true }


// NewManager returns new instance of the memory manager
func NewManager(policyName string, machineInfo *cadvisorapi.MachineInfo, nodeAllocatableReservation v1.ResourceList, reservedMemory []kubeletconfig.MemoryReservation, stateFileDirectory string, affinity topologymanager.Store) (Manager, error) {
	var policy Policy

	switch policyType(policyName) {

	case policyTypeNone:
		policy = NewPolicyNone()

	case policyTypeStatic:
		systemReserved, err := getSystemReservedMemory(machineInfo, nodeAllocatableReservation, reservedMemory)
		if err != nil {
			return nil, err
		}

		policy, err = NewPolicyStatic(machineInfo, systemReserved, affinity)
		if err != nil {
			return nil, err
		}

	default:
		return nil, fmt.Errorf("unknown policy: \"%s\"", policyName)
	}

	manager := &manager{
		policy:             policy,
		stateFileDirectory: stateFileDirectory,
	}
	manager.sourcesReady = &sourcesReadyStub{}
	return manager, nil
}


// Start starts the memory manager under the kubelet and calls policy start
func (m *manager) Start(activePods ActivePodsFunc, sourcesReady config.SourcesReady, podStatusProvider status.PodStatusProvider, containerRuntime runtimeService, initialContainers containermap.ContainerMap) error {
	klog.InfoS("Starting memorymanager", "policy", m.policy.Name())
	m.sourcesReady = sourcesReady
	m.activePods = activePods
	m.podStatusProvider = podStatusProvider
	m.containerRuntime = containerRuntime
	m.containerMap = initialContainers

	stateImpl, err := state.NewCheckpointState(m.stateFileDirectory, memoryManagerStateFileName, m.policy.Name())
	if err != nil {
		klog.ErrorS(err, "Could not initialize checkpoint manager, please drain node and remove policy state file")
		return err
	}
	m.state = stateImpl

	err = m.policy.Start(m.state)
	if err != nil {
		klog.ErrorS(err, "Policy start error")
		return err
	}

	m.allocatableMemory = m.policy.GetAllocatableMemory(m.state)

	return nil
}


// AddContainer saves the value of requested memory for the guaranteed pod under the state and set memory affinity according to the topolgy manager
func (m *manager) AddContainer(pod *v1.Pod, container *v1.Container, containerID string) {
	m.Lock()
	defer m.Unlock()

	m.containerMap.Add(string(pod.UID), container.Name, containerID)

	// Since we know that each init container always runs to completion before
	// the next container starts, we can safely remove references to any previously
	// started init containers. This will free up the memory from these init containers
	// for use in other pods. If the current container happens to be an init container,
	// we skip deletion of it until the next container is added, and this is called again.
	for _, initContainer := range pod.Spec.InitContainers {
		if initContainer.Name == container.Name {
			break
		}

		m.policyRemoveContainerByRef(string(pod.UID), initContainer.Name)
	}
}


// GetMemoryNUMANodes provides NUMA nodes that used to allocate the container memory
func (m *manager) GetMemoryNUMANodes(pod *v1.Pod, container *v1.Container) sets.Int {
	// Get NUMA node affinity of blocks assigned to the container during Allocate()
	numaNodes := sets.NewInt()
	for _, block := range m.state.GetMemoryBlocks(string(pod.UID), container.Name) {
		for _, nodeID := range block.NUMAAffinity {
			// avoid nodes duplication when hugepages and memory blocks pinned to the same NUMA node
			numaNodes.Insert(nodeID)
		}
	}

	if numaNodes.Len() == 0 {
		klog.V(5).InfoS("No allocation is available", "pod", klog.KObj(pod), "containerName", container.Name)
		return nil
	}

	klog.InfoS("Memory affinity", "pod", klog.KObj(pod), "containerName", container.Name, "numaNodes", numaNodes)
	return numaNodes
}


// Allocate is called to pre-allocate memory resources during Pod admission.
func (m *manager) Allocate(pod *v1.Pod, container *v1.Container) error {
	// The pod is during the admission phase. We need to save the pod to avoid it
	// being cleaned before the admission ended
	m.setPodPendingAdmission(pod)

	// Garbage collect any stranded resources before allocation
	m.removeStaleState()

	m.Lock()
	defer m.Unlock()

	// Call down into the policy to assign this container memory if required.
	if err := m.policy.Allocate(m.state, pod, container); err != nil {
		klog.ErrorS(err, "Allocate error")
		return err
	}
	return nil
}


// RemoveContainer removes the container from the state
func (m *manager) RemoveContainer(containerID string) error {
	m.Lock()
	defer m.Unlock()

	// if error appears it means container entry already does not exist under the container map
	podUID, containerName, err := m.containerMap.GetContainerRef(containerID)
	if err != nil {
		klog.InfoS("Failed to get container from container map", "containerID", containerID, "err", err)
		return nil
	}

	m.policyRemoveContainerByRef(podUID, containerName)

	return nil
}


func (m *manager) State() state.Reader

// State returns the state of the manager
func (m *manager) State() state.Reader {
	return m.state
}


// GetPodTopologyHints returns the topology hints for the topology manager
func (m *manager) GetPodTopologyHints(pod *v1.Pod) map[string][]topologymanager.TopologyHint {
	// The pod is during the admission phase. We need to save the pod to avoid it
	// being cleaned before the admission ended
	m.setPodPendingAdmission(pod)

	// Garbage collect any stranded resources before providing TopologyHints
	m.removeStaleState()
	// Delegate to active policy
	return m.policy.GetPodTopologyHints(m.state, pod)
}


// GetTopologyHints returns the topology hints for the topology manager
func (m *manager) GetTopologyHints(pod *v1.Pod, container *v1.Container) map[string][]topologymanager.TopologyHint {
	// The pod is during the admission phase. We need to save the pod to avoid it
	// being cleaned before the admission ended
	m.setPodPendingAdmission(pod)

	// Garbage collect any stranded resources before providing TopologyHints
	m.removeStaleState()
	// Delegate to active policy
	return m.policy.GetTopologyHints(m.state, pod, container)
}


func (m *manager) removeStaleState()

// TODO: move the method to the upper level, to re-use it under the CPU and memory managers
func (m *manager) removeStaleState() {
	// Only once all sources are ready do we attempt to remove any stale state.
	// This ensures that the call to `m.activePods()` below will succeed with
	// the actual active pods list.
	if !m.sourcesReady.AllReady() {
		return
	}

	// We grab the lock to ensure that no new containers will grab memory block while
	// executing the code below. Without this lock, its possible that we end up
	// removing state that is newly added by an asynchronous call to
	// AddContainer() during the execution of this code.
	m.Lock()
	defer m.Unlock()

	// Get the list of admitted and active pods.
	activeAndAdmittedPods := m.activePods()
	if m.pendingAdmissionPod != nil {
		activeAndAdmittedPods = append(activeAndAdmittedPods, m.pendingAdmissionPod)
	}

	// Build a list of (podUID, containerName) pairs for all containers in all active Pods.
	activeContainers := make(map[string]map[string]struct{})
	for _, pod := range activeAndAdmittedPods {
		activeContainers[string(pod.UID)] = make(map[string]struct{})
		for _, container := range append(pod.Spec.InitContainers, pod.Spec.Containers...) {
			activeContainers[string(pod.UID)][container.Name] = struct{}{}
		}
	}

	// Loop through the MemoryManager state. Remove any state for containers not
	// in the `activeContainers` list built above.
	assignments := m.state.GetMemoryAssignments()
	for podUID := range assignments {
		for containerName := range assignments[podUID] {
			if _, ok := activeContainers[podUID][containerName]; !ok {
				klog.InfoS("RemoveStaleState removing state", "podUID", podUID, "containerName", containerName)
				m.policyRemoveContainerByRef(podUID, containerName)
			}
		}
	}

	m.containerMap.Visit(func(podUID, containerName, containerID string) {
		if _, ok := activeContainers[podUID][containerName]; !ok {
			klog.InfoS("RemoveStaleState removing state", "podUID", podUID, "containerName", containerName)
			m.policyRemoveContainerByRef(podUID, containerName)
		}
	})
}


func (m *manager) policyRemoveContainerByRef(podUID string, containerName string) {
	m.policy.RemoveContainer(m.state, podUID, containerName)
	m.containerMap.RemoveByContainerRef(podUID, containerName)
}


func getTotalMemoryTypeReserved(machineInfo *cadvisorapi.MachineInfo, reservedMemory []kubeletconfig.MemoryReservation) (map[v1.ResourceName]resource.Quantity, error) {
	totalMemoryType := map[v1.ResourceName]resource.Quantity{}

	numaNodes := map[int]bool{}
	for _, numaNode := range machineInfo.Topology {
		numaNodes[numaNode.Id] = true
	}

	for _, reservation := range reservedMemory {
		if !numaNodes[int(reservation.NumaNode)] {
			return nil, fmt.Errorf("the reserved memory configuration references a NUMA node %d that does not exist on this machine", reservation.NumaNode)
		}

		for resourceName, q := range reservation.Limits {
			if value, ok := totalMemoryType[resourceName]; ok {
				q.Add(value)
			}
			totalMemoryType[resourceName] = q
		}
	}

	return totalMemoryType, nil
}


func validateReservedMemory(machineInfo *cadvisorapi.MachineInfo, nodeAllocatableReservation v1.ResourceList, reservedMemory []kubeletconfig.MemoryReservation) error {
	totalMemoryType, err := getTotalMemoryTypeReserved(machineInfo, reservedMemory)
	if err != nil {
		return err
	}

	commonMemoryTypeSet := make(map[v1.ResourceName]bool)
	for resourceType := range totalMemoryType {
		commonMemoryTypeSet[resourceType] = true
	}

	for resourceType := range nodeAllocatableReservation {
		if !(corev1helper.IsHugePageResourceName(resourceType) || resourceType == v1.ResourceMemory) {
			continue
		}
		commonMemoryTypeSet[resourceType] = true
	}

	for resourceType := range commonMemoryTypeSet {
		nodeAllocatableMemory := resource.NewQuantity(0, resource.DecimalSI)
		if memValue, set := nodeAllocatableReservation[resourceType]; set {
			nodeAllocatableMemory.Add(memValue)
		}

		reservedMemory := resource.NewQuantity(0, resource.DecimalSI)
		if memValue, set := totalMemoryType[resourceType]; set {
			reservedMemory.Add(memValue)
		}

		if !(*nodeAllocatableMemory).Equal(*reservedMemory) {
			return fmt.Errorf("the total amount %q of type %q is not equal to the value %q determined by Node Allocatable feature", reservedMemory.String(), resourceType, nodeAllocatableMemory.String())
		}
	}

	return nil
}


func convertReserved(machineInfo *cadvisorapi.MachineInfo, reservedMemory []kubeletconfig.MemoryReservation) (systemReservedMemory, error) {
	reservedMemoryConverted := make(map[int]map[v1.ResourceName]uint64)
	for _, node := range machineInfo.Topology {
		reservedMemoryConverted[node.Id] = make(map[v1.ResourceName]uint64)
	}

	for _, reservation := range reservedMemory {
		for resourceName, q := range reservation.Limits {
			val, success := q.AsInt64()
			if !success {
				return nil, fmt.Errorf("could not covert a variable of type Quantity to int64")
			}
			reservedMemoryConverted[int(reservation.NumaNode)][resourceName] = uint64(val)
		}
	}

	return reservedMemoryConverted, nil
}


func getSystemReservedMemory(machineInfo *cadvisorapi.MachineInfo, nodeAllocatableReservation v1.ResourceList, reservedMemory []kubeletconfig.MemoryReservation) (systemReservedMemory, error) {
	if err := validateReservedMemory(machineInfo, nodeAllocatableReservation, reservedMemory); err != nil {
		return nil, err
	}

	reservedMemoryConverted, err := convertReserved(machineInfo, reservedMemory)
	if err != nil {
		return nil, err
	}

	return reservedMemoryConverted, nil
}


// GetAllocatableMemory returns the amount of allocatable memory for each NUMA node
func (m *manager) GetAllocatableMemory() []state.Block {
	return m.allocatableMemory
}


// GetMemory returns the memory allocated by a container from NUMA nodes
func (m *manager) GetMemory(podUID, containerName string) []state.Block {
	return m.state.GetMemoryBlocks(podUID, containerName)
}


func (m *manager) setPodPendingAdmission(pod *v1.Pod) {
	m.Lock()
	defer m.Unlock()

	m.pendingAdmissionPod = pod
}


func NewPolicyNone() Policy

// NewPolicyNone returns new none policy instance
func NewPolicyNone() Policy {
	return &none{}
}


func (p *none) Name() string

func (p *none) Name() string {
	return string(policyTypeNone)
}


func (p *none) Start(s state.State) error {
	return nil
}


// Allocate call is idempotent
func (p *none) Allocate(s state.State, pod *v1.Pod, container *v1.Container) error {
	return nil
}


// RemoveContainer call is idempotent
func (p *none) RemoveContainer(s state.State, podUID string, containerName string) {
}


// GetTopologyHints implements the topologymanager.HintProvider Interface
// and is consulted to achieve NUMA aware resource alignment among this
// and other resource controllers.
func (p *none) GetTopologyHints(s state.State, pod *v1.Pod, container *v1.Container) map[string][]topologymanager.TopologyHint {
	return nil
}


// GetPodTopologyHints implements the topologymanager.HintProvider Interface
// and is consulted to achieve NUMA aware resource alignment among this
// and other resource controllers.
func (p *none) GetPodTopologyHints(s state.State, pod *v1.Pod) map[string][]topologymanager.TopologyHint {
	return nil
}


// GetAllocatableMemory returns the amount of allocatable memory for each NUMA node
func (p *none) GetAllocatableMemory(s state.State) []state.Block {
	return []state.Block{}
}


// NewPolicyStatic returns new static policy instance
func NewPolicyStatic(machineInfo *cadvisorapi.MachineInfo, reserved systemReservedMemory, affinity topologymanager.Store) (Policy, error) {
	var totalSystemReserved uint64
	for _, node := range reserved {
		if _, ok := node[v1.ResourceMemory]; !ok {
			continue
		}
		totalSystemReserved += node[v1.ResourceMemory]
	}

	// check if we have some reserved memory for the system
	if totalSystemReserved <= 0 {
		return nil, fmt.Errorf("[memorymanager] you should specify the system reserved memory")
	}

	return &staticPolicy{
		machineInfo:                  machineInfo,
		systemReserved:               reserved,
		affinity:                     affinity,
		initContainersReusableMemory: reusableMemory{},
	}, nil
}


func (p *staticPolicy) Name() string

func (p *staticPolicy) Name() string {
	return string(policyTypeStatic)
}


func (p *staticPolicy) Start(s state.State) error {
	if err := p.validateState(s); err != nil {
		klog.ErrorS(err, "Invalid state, please drain node and remove policy state file")
		return err
	}
	return nil
}