convertResourceEphemeralStorageToString

// convertResourceEphemeralStorageToString converts ephemeral storage value to the format of divisor and returns
// ceiling of the value.
func convertResourceEphemeralStorageToString(ephemeralStorage *resource.Quantity, divisor resource.Quantity) (string, error) {
	m := int64(math.Ceil(float64(ephemeralStorage.Value()) / float64(divisor.Value())))
	return strconv.FormatInt(m, 10), nil
}


IsStandardContainerResourceName

// IsStandardContainerResourceName returns true if the container can make a resource request
// for the specified resource
func IsStandardContainerResourceName(str string) bool {
	return standardContainerResources.Has(str) || IsHugePageResourceName(corev1.ResourceName(str))
}


IsHugePageResourceName

// IsHugePageResourceName returns true if the resource name has the huge page
// resource prefix.
func IsHugePageResourceName(name corev1.ResourceName) bool {
	return strings.HasPrefix(string(name), corev1.ResourceHugePagesPrefix)
}


LookupContainerPortNumberByServicePort

// LookupContainerPortNumberByServicePort implements
// the handling of resolving container named port, as well as ignoring targetPort when clusterIP=None
// It returns an error when a named port can't find a match (with -1 returned), or when the service does not
// declare such port (with the input port number returned).
func LookupContainerPortNumberByServicePort(svc v1.Service, pod v1.Pod, port int32) (int32, error) {
	for _, svcportspec := range svc.Spec.Ports {
		if svcportspec.Port != port {
			continue
		}
		if svc.Spec.ClusterIP == v1.ClusterIPNone {
			return port, nil
		}
		if svcportspec.TargetPort.Type == intstr.Int {
			if svcportspec.TargetPort.IntValue() == 0 {
				// targetPort is omitted, and the IntValue() would be zero
				return svcportspec.Port, nil
			}
			return int32(svcportspec.TargetPort.IntValue()), nil
		}
		return LookupContainerPortNumberByName(pod, svcportspec.TargetPort.String())
	}
	return port, fmt.Errorf("Service %s does not have a service port %d", svc.Name, port)
}


LookupServicePortNumberByName

// LookupServicePortNumberByName find service port number by its named port name
func LookupServicePortNumberByName(svc v1.Service, name string) (int32, error) {
	for _, svcportspec := range svc.Spec.Ports {
		if svcportspec.Name == name {
			return svcportspec.Port, nil
		}
	}

	return int32(-1), fmt.Errorf("Service '%s' does not have a named port '%s'", svc.Name, name)
}


// SortInts64 sorts []int64 in increasing order
func SortInts64(a []int64) { sort.Slice(a, func(i, j int) bool { return a[i] < a[j] }) }


ContainsString

// ContainsString checks if a given slice of strings contains the provided string.
// If a modifier func is provided, it is called with the slice item before the comparation.
func ContainsString(slice []string, s string, modifier func(s string) string) bool {
	for _, item := range slice {
		if item == s {
			return true
		}
		if modifier != nil && modifier(item) == s {
			return true
		}
	}
	return false
}


ToSet

// ToSet returns a single slice containing the unique values from one or more slices. The order of the items in the
// result is not guaranteed.
func ToSet[T comparable](slices ...[]T) []T {
	if len(slices) == 0 {
		return nil
	}
	m := map[T]struct{}{}
	for _, slice := range slices {
		for _, value := range slice {
			m[value] = struct{}{}
		}
	}
	result := []T{}
	for k := range m {
		result = append(result, k)
	}
	return result
}


IsDefaultAnnotationText

// IsDefaultAnnotationText returns a pretty Yes/No String if
// the annotation is set
func IsDefaultAnnotationText(obj metav1.ObjectMeta) string {
	if obj.Annotations[IsDefaultStorageClassAnnotation] == "true" {
		return "Yes"
	}
	if obj.Annotations[BetaIsDefaultStorageClassAnnotation] == "true" {
		return "Yes"
	}

	return "No"
}


GetAccessModesAsString

// GetAccessModesAsString returns a string representation of an array of access modes.
// modes, when present, are always in the same order: RWO,ROX,RWX,RWOP.
func GetAccessModesAsString(modes []v1.PersistentVolumeAccessMode) string {
	modes = removeDuplicateAccessModes(modes)
	modesStr := []string{}
	if ContainsAccessMode(modes, v1.ReadWriteOnce) {
		modesStr = append(modesStr, "RWO")
	}
	if ContainsAccessMode(modes, v1.ReadOnlyMany) {
		modesStr = append(modesStr, "ROX")
	}
	if ContainsAccessMode(modes, v1.ReadWriteMany) {
		modesStr = append(modesStr, "RWX")
	}
	if ContainsAccessMode(modes, v1.ReadWriteOncePod) {
		modesStr = append(modesStr, "RWOP")
	}
	return strings.Join(modesStr, ",")
}


removeDuplicateAccessModes

// removeDuplicateAccessModes returns an array of access modes without any duplicates
func removeDuplicateAccessModes(modes []v1.PersistentVolumeAccessMode) []v1.PersistentVolumeAccessMode {
	accessModes := []v1.PersistentVolumeAccessMode{}
	for _, m := range modes {
		if !ContainsAccessMode(accessModes, m) {
			accessModes = append(accessModes, m)
		}
	}
	return accessModes
}


ContainsAccessMode

func ContainsAccessMode(modes []v1.PersistentVolumeAccessMode, mode v1.PersistentVolumeAccessMode) bool {
	for _, m := range modes {
		if m == mode {
			return true
		}
	}
	return false
}


GetPersistentVolumeClass

// GetPersistentVolumeClass returns StorageClassName.
func GetPersistentVolumeClass(volume *v1.PersistentVolume) string {
	// Use beta annotation first
	if class, found := volume.Annotations[v1.BetaStorageClassAnnotation]; found {
		return class
	}

	return volume.Spec.StorageClassName
}


GetPersistentVolumeClaimClass

// GetPersistentVolumeClaimClass returns StorageClassName. If no storage class was
// requested, it returns "".
func GetPersistentVolumeClaimClass(claim *v1.PersistentVolumeClaim) string {
	// Use beta annotation first
	if class, found := claim.Annotations[v1.BetaStorageClassAnnotation]; found {
		return class
	}

	if claim.Spec.StorageClassName != nil {
		return *claim.Spec.StorageClassName
	}

	return ""
}


Add

func (g CommandGroups) Add(c *cobra.Command) {
	for _, group := range g {
		c.AddCommand(group.Commands...)
	}
}


Has

func (g CommandGroups) Has(c *cobra.Command) bool {
	for _, group := range g {
		for _, command := range group.Commands {
			if command == c {
				return true
			}
		}
	}
	return false
}


AddAdditionalCommands

func AddAdditionalCommands(g CommandGroups, message string, cmds []*cobra.Command) CommandGroups {
	group := CommandGroup{Message: message}
	for _, c := range cmds {
		// Don't show commands that have no short description
		if !g.Has(c) && len(c.Short) != 0 {
			group.Commands = append(group.Commands, c)
		}
	}
	if len(group.Commands) == 0 {
		return g
	}
	return append(g, group)
}


NewHelpFlagPrinter

// NewHelpFlagPrinter will initialize a HelpFlagPrinter given the
// i/o writer
func NewHelpFlagPrinter(out io.Writer, wrapLimit uint) *HelpFlagPrinter {
	return &HelpFlagPrinter{
		wrapLimit: wrapLimit,
		out:       out,
	}
}


PrintHelpFlag

// PrintHelpFlag will beautify the help flags and print it out to p.out
func (p *HelpFlagPrinter) PrintHelpFlag(flag *flag.Flag) {
	formatBuf := new(bytes.Buffer)
	writeFlag(formatBuf, flag)

	wrappedStr := formatBuf.String()
	flagAndUsage := strings.Split(formatBuf.String(), "\n")
	flagStr := flagAndUsage[0]

	// if the flag usage is longer than one line, wrap it again
	if len(flagAndUsage) > 1 {
		nextLines := strings.Join(flagAndUsage[1:], " ")
		wrappedUsages := wordwrap.WrapString(nextLines, p.wrapLimit-offset)
		wrappedStr = flagStr + "\n" + wrappedUsages
	}
	appendTabStr := strings.ReplaceAll(wrappedStr, "\n", "\n\t")

	fmt.Fprintf(p.out, appendTabStr+"\n\n")
}


writeFlag

// writeFlag will output the help flag based
// on the format provided by getFlagFormat to i/o writer
func writeFlag(out io.Writer, f *flag.Flag) {
	deprecated := ""
	if f.Deprecated != "" {
		deprecated = fmt.Sprintf(" (DEPRECATED: %s)", f.Deprecated)
	}
	fmt.Fprintf(out, getFlagFormat(f), f.Shorthand, f.Name, f.DefValue, f.Usage, deprecated)
}


RenderNode

// render markdown to text
func (r *ASCIIRenderer) RenderNode(w io.Writer, node *blackfriday.Node, entering bool) blackfriday.WalkStatus {
	switch node.Type {
	case blackfriday.Text:
		raw := string(node.Literal)
		lines := strings.Split(raw, linebreak)
		for _, line := range lines {
			trimmed := strings.Trim(line, " \n\t")
			if len(trimmed) > 0 && trimmed[0] != '_' {
				w.Write([]byte(" "))
			}
			w.Write([]byte(trimmed))
		}
	case blackfriday.HorizontalRule, blackfriday.Hardbreak:
		w.Write([]byte(linebreak + "----------" + linebreak))
	case blackfriday.Code, blackfriday.CodeBlock:
		w.Write([]byte(linebreak))
		lines := []string{}
		for _, line := range strings.Split(string(node.Literal), linebreak) {
			trimmed := strings.Trim(line, " \t")
			indented := r.Indentation + trimmed
			lines = append(lines, indented)
		}
		w.Write([]byte(strings.Join(lines, linebreak)))
	case blackfriday.Image:
		w.Write(node.LinkData.Destination)
	case blackfriday.Link:
		w.Write([]byte(" "))
		w.Write(node.LinkData.Destination)
	case blackfriday.Paragraph:
		if r.listLevel == 0 {
			w.Write([]byte(linebreak))
		}
	case blackfriday.List:
		if entering {
			w.Write([]byte(linebreak))
			r.listLevel++
		} else {
			r.listLevel--
			r.listItemCount = 0
		}
	case blackfriday.Item:
		if entering {
			r.listItemCount++
			for i := 0; uint(i) < r.listLevel; i++ {
				w.Write([]byte(r.Indentation))
			}
			if node.ListFlags&blackfriday.ListTypeOrdered != 0 {
				w.Write([]byte(fmt.Sprintf("%d. ", r.listItemCount)))
			} else {
				w.Write([]byte("* "))
			}
		} else {
			w.Write([]byte(linebreak))
		}
	default:
		normalText(w, node.Literal)
	}
	return blackfriday.GoToNext
}


normalText

func normalText(w io.Writer, text []byte) {
	w.Write([]byte(strings.Trim(string(text), " \n\t")))
}


RenderHeader

// RenderHeader writes document preamble and TOC if requested.
func (r *ASCIIRenderer) RenderHeader(w io.Writer, ast *blackfriday.Node) {

}


RenderFooter

// RenderFooter writes document footer.
func (r *ASCIIRenderer) RenderFooter(w io.Writer, ast *blackfriday.Node) {
	io.WriteString(w, "\n")
}


LongDesc

// LongDesc normalizes a command's long description to follow the conventions.
func LongDesc(s string) string {
	if len(s) == 0 {
		return s
	}
	return normalizer{s}.heredoc().markdown().trim().string
}


Examples

// Examples normalizes a command's examples to follow the conventions.
func Examples(s string) string {
	if len(s) == 0 {
		return s
	}
	return normalizer{s}.trim().indent().string
}


Normalize

// Normalize perform all required normalizations on a given command.
func Normalize(cmd *cobra.Command) *cobra.Command {
	if len(cmd.Long) > 0 {
		cmd.Long = LongDesc(cmd.Long)
	}
	if len(cmd.Example) > 0 {
		cmd.Example = Examples(cmd.Example)
	}
	return cmd
}


NormalizeAll

// NormalizeAll perform all required normalizations in the entire command tree.
func NormalizeAll(cmd *cobra.Command) *cobra.Command {
	if cmd.HasSubCommands() {
		for _, subCmd := range cmd.Commands() {
			NormalizeAll(subCmd)
		}
	}
	Normalize(cmd)
	return cmd
}


markdown

func (s normalizer) markdown() normalizer {
	bytes := []byte(s.string)
	formatted := blackfriday.Run(bytes, blackfriday.WithExtensions(blackfriday.NoIntraEmphasis), blackfriday.WithRenderer(&ASCIIRenderer{Indentation: Indentation}))
	s.string = string(formatted)
	return s
}


heredoc

func (s normalizer) heredoc() normalizer {
	s.string = heredoc.Doc(s.string)
	return s
}


trim

func (s normalizer) trim() normalizer {
	s.string = strings.TrimSpace(s.string)
	return s
}


indent

func (s normalizer) indent() normalizer {
	indentedLines := []string{}
	for _, line := range strings.Split(s.string, "\n") {
		trimmed := strings.TrimSpace(line)
		indented := Indentation + trimmed
		indentedLines = append(indentedLines, indented)
	}
	s.string = strings.Join(indentedLines, "\n")
	return s
}


ActsAsRootCommand

func ActsAsRootCommand(cmd *cobra.Command, filters []string, groups ...CommandGroup) FlagExposer {
	if cmd == nil {
		panic("nil root command")
	}
	templater := &templater{
		RootCmd:       cmd,
		UsageTemplate: MainUsageTemplate(),
		HelpTemplate:  MainHelpTemplate(),
		CommandGroups: groups,
		Filtered:      filters,
	}
	cmd.SetFlagErrorFunc(templater.FlagErrorFunc())
	cmd.SilenceUsage = true
	cmd.SetUsageFunc(templater.UsageFunc())
	cmd.SetHelpFunc(templater.HelpFunc())
	return templater
}


UseOptionsTemplates

func UseOptionsTemplates(cmd *cobra.Command) {
	templater := &templater{
		UsageTemplate: OptionsUsageTemplate(),
		HelpTemplate:  OptionsHelpTemplate(),
	}
	cmd.SetUsageFunc(templater.UsageFunc())
	cmd.SetHelpFunc(templater.HelpFunc())
}


FlagErrorFunc

func (templater *templater) FlagErrorFunc(exposedFlags ...string) func(*cobra.Command, error) error {
	return func(c *cobra.Command, err error) error {
		c.SilenceUsage = true
		switch c.CalledAs() {
		case "options":
			return fmt.Errorf("%s\nRun '%s' without flags.", err, c.CommandPath())
		default:
			return fmt.Errorf("%s\nSee '%s --help' for usage.", err, c.CommandPath())
		}
	}
}


ExposeFlags

func (templater *templater) ExposeFlags(cmd *cobra.Command, flags ...string) FlagExposer {
	cmd.SetUsageFunc(templater.UsageFunc(flags...))
	return templater
}


HelpFunc

func (templater *templater) HelpFunc() func(*cobra.Command, []string) {
	return func(c *cobra.Command, s []string) {
		t := template.New("help")
		t.Funcs(templater.templateFuncs())
		template.Must(t.Parse(templater.HelpTemplate))
		out := term.NewResponsiveWriter(c.OutOrStdout())
		err := t.Execute(out, c)
		if err != nil {
			c.Println(err)
		}
	}
}


UsageFunc

func (templater *templater) UsageFunc(exposedFlags ...string) func(*cobra.Command) error {
	return func(c *cobra.Command) error {
		t := template.New("usage")
		t.Funcs(templater.templateFuncs(exposedFlags...))
		template.Must(t.Parse(templater.UsageTemplate))
		out := term.NewResponsiveWriter(c.OutOrStderr())
		return t.Execute(out, c)
	}
}


templateFuncs

func (templater *templater) templateFuncs(exposedFlags ...string) template.FuncMap {
	return template.FuncMap{
		"trim":                strings.TrimSpace,
		"trimRight":           func(s string) string { return strings.TrimRightFunc(s, unicode.IsSpace) },
		"trimLeft":            func(s string) string { return strings.TrimLeftFunc(s, unicode.IsSpace) },
		"gt":                  cobra.Gt,
		"eq":                  cobra.Eq,
		"rpad":                rpad,
		"appendIfNotPresent":  appendIfNotPresent,
		"flagsNotIntersected": flagsNotIntersected,
		"visibleFlags":        visibleFlags,
		"flagsUsages":         flagsUsages,
		"cmdGroups":           templater.cmdGroups,
		"cmdGroupsString":     templater.cmdGroupsString,
		"rootCmd":             templater.rootCmdName,
		"isRootCmd":           templater.isRootCmd,
		"optionsCmdFor":       templater.optionsCmdFor,
		"usageLine":           templater.usageLine,
		"exposed": func(c *cobra.Command) *flag.FlagSet {
			exposed := flag.NewFlagSet("exposed", flag.ContinueOnError)
			if len(exposedFlags) > 0 {
				for _, name := range exposedFlags {
					if flag := c.Flags().Lookup(name); flag != nil {
						exposed.AddFlag(flag)
					}
				}
			}
			return exposed
		},
	}
}


cmdGroups

func (templater *templater) cmdGroups(c *cobra.Command, all []*cobra.Command) []CommandGroup {
	if len(templater.CommandGroups) > 0 && c == templater.RootCmd {
		all = filter(all, templater.Filtered...)
		return AddAdditionalCommands(templater.CommandGroups, "Other Commands:", all)
	}
	all = filter(all, "options")
	return []CommandGroup{
		{
			Message:  "Available Commands:",
			Commands: all,
		},
	}
}


cmdGroupsString

func (t *templater) cmdGroupsString(c *cobra.Command) string {
	groups := []string{}
	for _, cmdGroup := range t.cmdGroups(c, c.Commands()) {
		cmds := []string{cmdGroup.Message}
		for _, cmd := range cmdGroup.Commands {
			if cmd.IsAvailableCommand() {
				cmds = append(cmds, "  "+rpad(cmd.Name(), cmd.NamePadding())+"   "+cmd.Short)
			}
		}
		groups = append(groups, strings.Join(cmds, "\n"))
	}
	return strings.Join(groups, "\n\n")
}


rootCmdName

func (t *templater) rootCmdName(c *cobra.Command) string {
	return t.rootCmd(c).CommandPath()
}


isRootCmd

func (t *templater) isRootCmd(c *cobra.Command) bool {
	return t.rootCmd(c) == c
}


parents

func (t *templater) parents(c *cobra.Command) []*cobra.Command {
	parents := []*cobra.Command{c}
	for current := c; !t.isRootCmd(current) && current.HasParent(); {
		current = current.Parent()
		parents = append(parents, current)
	}
	return parents
}


rootCmd

func (t *templater) rootCmd(c *cobra.Command) *cobra.Command {
	if c != nil && !c.HasParent() {
		return c
	}
	if t.RootCmd == nil {
		panic("nil root cmd")
	}
	return t.RootCmd
}


optionsCmdFor

func (t *templater) optionsCmdFor(c *cobra.Command) string {
	if !c.Runnable() {
		return ""
	}
	rootCmdStructure := t.parents(c)
	for i := len(rootCmdStructure) - 1; i >= 0; i-- {
		cmd := rootCmdStructure[i]
		if _, _, err := cmd.Find([]string{"options"}); err == nil {
			return cmd.CommandPath() + " options"
		}
	}
	return ""
}


usageLine

func (t *templater) usageLine(c *cobra.Command) string {
	usage := c.UseLine()
	suffix := "[options]"
	if c.HasFlags() && !strings.Contains(usage, suffix) {
		usage += " " + suffix
	}
	return usage
}


flagsUsages

// flagsUsages will print out the kubectl help flags
func flagsUsages(f *flag.FlagSet) (string, error) {
	flagBuf := new(bytes.Buffer)
	wrapLimit, err := term.GetWordWrapperLimit()
	if err != nil {
		wrapLimit = 0
	}
	printer := NewHelpFlagPrinter(flagBuf, wrapLimit)

	f.VisitAll(func(flag *flag.Flag) {
		if flag.Hidden {
			return
		}
		printer.PrintHelpFlag(flag)
	})

	return flagBuf.String(), nil
}


getFlagFormat

// getFlagFormat will output the flag format
func getFlagFormat(f *flag.Flag) string {
	var format string
	format = "--%s=%s:\n%s%s"
	if f.Value.Type() == "string" {
		format = "--%s='%s':\n%s%s"
	}

	if len(f.Shorthand) > 0 {
		format = "    -%s, " + format
	} else {
		format = "    %s" + format
	}

	return format
}


rpad

func rpad(s string, padding int) string {
	template := fmt.Sprintf("%%-%ds", padding)
	return fmt.Sprintf(template, s)
}


appendIfNotPresent

func appendIfNotPresent(s, stringToAppend string) string {
	if strings.Contains(s, stringToAppend) {
		return s
	}
	return s + " " + stringToAppend
}


flagsNotIntersected

func flagsNotIntersected(l *flag.FlagSet, r *flag.FlagSet) *flag.FlagSet {
	f := flag.NewFlagSet("notIntersected", flag.ContinueOnError)
	l.VisitAll(func(flag *flag.Flag) {
		if r.Lookup(flag.Name) == nil {
			f.AddFlag(flag)
		}
	})
	return f
}


visibleFlags

func visibleFlags(l *flag.FlagSet) *flag.FlagSet {
	hidden := "help"
	f := flag.NewFlagSet("visible", flag.ContinueOnError)
	l.VisitAll(func(flag *flag.Flag) {
		if flag.Name != hidden {
			f.AddFlag(flag)
		}
	})
	return f
}


filter

func filter(cmds []*cobra.Command, names ...string) []*cobra.Command {
	out := []*cobra.Command{}
	for _, c := range cmds {
		if c.Hidden {
			continue
		}
		skip := false
		for _, name := range names {
			if name == c.Name() {
				skip = true
				break
			}
		}
		if skip {
			continue
		}
		out = append(out, c)
	}
	return out
}


MainHelpTemplate

// MainHelpTemplate if the template for 'help' used by most commands.
func MainHelpTemplate() string {
	return `{{with or .Long .Short }}{{. | trim}}{{end}}{{if or .Runnable .HasSubCommands}}{{.UsageString}}{{end}}`
}


MainUsageTemplate

// MainUsageTemplate if the template for 'usage' used by most commands.
func MainUsageTemplate() string {
	sections := []string{
		"\n\n",
		SectionVars,
		SectionAliases,
		SectionExamples,
		SectionSubcommands,
		SectionFlags,
		SectionUsage,
		SectionTipsHelp,
		SectionTipsGlobalOptions,
	}
	return strings.TrimRightFunc(strings.Join(sections, ""), unicode.IsSpace)
}


OptionsHelpTemplate

// OptionsHelpTemplate if the template for 'help' used by the 'options' command.
func OptionsHelpTemplate() string {
	return ""
}


OptionsUsageTemplate

// OptionsUsageTemplate if the template for 'usage' used by the 'options' command.
func OptionsUsageTemplate() string {
	return `{{ if .HasInheritedFlags}}The following options can be passed to any command:

{{flagsUsages .InheritedFlags}}{{end}}`
}


GetSize

// GetSize returns the current size of the user's terminal. If it isn't a terminal,
// nil is returned.
func (t TTY) GetSize() *remotecommand.TerminalSize {
	outFd, isTerminal := term.GetFdInfo(t.Out)
	if !isTerminal {
		return nil
	}
	return GetSize(outFd)
}


// GetSize returns the current size of the terminal associated with fd.
func GetSize(fd uintptr) *remotecommand.TerminalSize {
	winsize, err := term.GetWinsize(fd)
	if err != nil {
		runtime.HandleError(fmt.Errorf("unable to get terminal size: %v", err))
		return nil
	}

	return &remotecommand.TerminalSize{Width: winsize.Width, Height: winsize.Height}
}


MonitorSize

// MonitorSize monitors the terminal's size. It returns a TerminalSizeQueue primed with
// initialSizes, or nil if there's no TTY present.
func (t *TTY) MonitorSize(initialSizes ...*remotecommand.TerminalSize) remotecommand.TerminalSizeQueue {
	outFd, isTerminal := term.GetFdInfo(t.Out)
	if !isTerminal {
		return nil
	}

	t.sizeQueue = &sizeQueue{
		t: *t,
		// make it buffered so we can send the initial terminal sizes without blocking, prior to starting
		// the streaming below
		resizeChan:   make(chan remotecommand.TerminalSize, len(initialSizes)),
		stopResizing: make(chan struct{}),
	}

	t.sizeQueue.monitorSize(outFd, initialSizes...)

	return t.sizeQueue
}


monitorSize

// monitorSize primes resizeChan with initialSizes and then monitors for resize events. With each
// new event, it sends the current terminal size to resizeChan.
func (s *sizeQueue) monitorSize(outFd uintptr, initialSizes ...*remotecommand.TerminalSize) {
	// send the initial sizes
	for i := range initialSizes {
		if initialSizes[i] != nil {
			s.resizeChan <- *initialSizes[i]
		}
	}

	resizeEvents := make(chan remotecommand.TerminalSize, 1)

	monitorResizeEvents(outFd, resizeEvents, s.stopResizing)

	// listen for resize events in the background
	go func() {
		defer runtime.HandleCrash()

		for {
			select {
			case size, ok := <-resizeEvents:
				if !ok {
					return
				}

				select {
				// try to send the size to resizeChan, but don't block
				case s.resizeChan <- size:
					// send successful
				default:
					// unable to send / no-op
				}
			case <-s.stopResizing:
				return
			}
		}
	}()
}


Next

// Next returns the new terminal size after the terminal has been resized. It returns nil when
// monitoring has been stopped.
func (s *sizeQueue) Next() *remotecommand.TerminalSize {
	size, ok := <-s.resizeChan
	if !ok {
		return nil
	}
	return &size
}


stop

// stop stops the background goroutine that is monitoring for terminal resizes.
func (s *sizeQueue) stop() {
	close(s.stopResizing)
}


monitorResizeEvents

// monitorResizeEvents spawns a goroutine that waits for SIGWINCH signals (these indicate the
// terminal has resized). After receiving a SIGWINCH, this gets the terminal size and tries to send
// it to the resizeEvents channel. The goroutine stops when the stop channel is closed.
func monitorResizeEvents(fd uintptr, resizeEvents chan<- remotecommand.TerminalSize, stop chan struct{}) {
	go func() {
		defer runtime.HandleCrash()

		winch := make(chan os.Signal, 1)
		signal.Notify(winch, unix.SIGWINCH)
		defer signal.Stop(winch)

		for {
			select {
			case <-winch:
				size := GetSize(fd)
				if size == nil {
					return
				}

				// try to send size
				select {
				case resizeEvents <- *size:
					// success
				default:
					// not sent
				}
			case <-stop:
				return
			}
		}
	}()
}


// monitorResizeEvents spawns a goroutine that periodically gets the terminal size and tries to send
// it to the resizeEvents channel if the size has changed. The goroutine stops when the stop channel
// is closed.
func monitorResizeEvents(fd uintptr, resizeEvents chan<- remotecommand.TerminalSize, stop chan struct{}) {
	go func() {
		defer runtime.HandleCrash()

		size := GetSize(fd)
		if size == nil {
			return
		}
		lastSize := *size

		for {
			// see if we need to stop running
			select {
			case <-stop:
				return
			default:
			}

			size := GetSize(fd)
			if size == nil {
				return
			}

			if size.Height != lastSize.Height || size.Width != lastSize.Width {
				lastSize.Height = size.Height
				lastSize.Width = size.Width
				resizeEvents <- *size
			}

			// sleep to avoid hot looping
			time.Sleep(250 * time.Millisecond)
		}
	}()
}


IsTerminalIn

// IsTerminalIn returns true if t.In is a terminal. Does not check /dev/tty
// even if TryDev is set.
func (t TTY) IsTerminalIn() bool {
	return IsTerminal(t.In)
}


IsTerminalOut

// IsTerminalOut returns true if t.Out is a terminal. Does not check /dev/tty
// even if TryDev is set.
func (t TTY) IsTerminalOut() bool {
	return IsTerminal(t.Out)
}


IsTerminal

// IsTerminal returns whether the passed object is a terminal or not
func IsTerminal(i interface{}) bool {
	_, terminal := term.GetFdInfo(i)
	return terminal
}


AllowsColorOutput

// AllowsColorOutput returns true if the specified writer is a terminal and
// the process environment indicates color output is supported and desired.
func AllowsColorOutput(w io.Writer) bool {
	if !IsTerminal(w) {
		return false
	}

	// https://en.wikipedia.org/wiki/Computer_terminal#Dumb_terminals
	if os.Getenv("TERM") == "dumb" {
		return false
	}

	// https://no-color.org/
	if _, nocolor := os.LookupEnv("NO_COLOR"); nocolor {
		return false
	}

	// On Windows WT_SESSION is set by the modern terminal component.
	// Older terminals have poor support for UTF-8, VT escape codes, etc.
	if runtime.GOOS == "windows" && os.Getenv("WT_SESSION") == "" {
		return false
	}

	return true
}


Safe

// Safe invokes the provided function and will attempt to ensure that when the
// function returns (or a termination signal is sent) that the terminal state
// is reset to the condition it was in prior to the function being invoked. If
// t.Raw is true the terminal will be put into raw mode prior to calling the function.
// If the input file descriptor is not a TTY and TryDev is true, the /dev/tty file
// will be opened (if available).
func (t TTY) Safe(fn SafeFunc) error {
	inFd, isTerminal := term.GetFdInfo(t.In)

	if !isTerminal && t.TryDev {
		if f, err := os.Open("/dev/tty"); err == nil {
			defer f.Close()
			inFd = f.Fd()
			isTerminal = term.IsTerminal(inFd)
		}
	}
	if !isTerminal {
		return fn()
	}

	var state *term.State
	var err error
	if t.Raw {
		state, err = term.MakeRaw(inFd)
	} else {
		state, err = term.SaveState(inFd)
	}
	if err != nil {
		return err
	}
	return interrupt.Chain(t.Parent, func() {
		if t.sizeQueue != nil {
			t.sizeQueue.stop()
		}

		term.RestoreTerminal(inFd, state)
	}).Run(fn)
}


NewResponsiveWriter

// NewResponsiveWriter creates a Writer that detects the column width of the
// terminal we are in, and adjusts every line width to fit and use recommended
// terminal sizes for better readability. Does proper word wrapping automatically.
//
//	if terminal width >= 120 columns		use 120 columns
//	if terminal width >= 100 columns		use 100 columns
//	if terminal width >=  80 columns		use  80 columns
//
// In case we're not in a terminal or if it's smaller than 80 columns width,
// doesn't do any wrapping.
func NewResponsiveWriter(w io.Writer) io.Writer {
	file, ok := w.(*os.File)
	if !ok {
		return w
	}
	fd := file.Fd()
	if !term.IsTerminal(fd) {
		return w
	}

	terminalSize := GetSize(fd)
	if terminalSize == nil {
		return w
	}
	limit := getTerminalLimitWidth(terminalSize)

	return NewWordWrapWriter(w, limit)
}


NewWordWrapWriter

// NewWordWrapWriter is a Writer that supports a limit of characters on every line
// and does auto word wrapping that respects that limit.
func NewWordWrapWriter(w io.Writer, limit uint) io.Writer {
	return &wordWrapWriter{
		limit:  limit,
		writer: w,
	}
}


getTerminalLimitWidth

func getTerminalLimitWidth(terminalSize *remotecommand.TerminalSize) uint {
	var limit uint
	switch {
	case terminalSize.Width >= 120:
		limit = 120
	case terminalSize.Width >= 100:
		limit = 100
	case terminalSize.Width >= 80:
		limit = 80
	}
	return limit
}


GetWordWrapperLimit

func GetWordWrapperLimit() (uint, error) {
	stdout := os.Stdout
	fd := stdout.Fd()
	if !term.IsTerminal(fd) {
		return 0, errors.New("file descriptor is not a terminal")
	}
	terminalSize := GetSize(fd)
	if terminalSize == nil {
		return 0, errors.New("terminal size is nil")
	}
	return getTerminalLimitWidth(terminalSize), nil
}


Write

func (w wordWrapWriter) Write(p []byte) (nn int, err error) {
	if w.limit == 0 {
		return w.writer.Write(p)
	}
	original := string(p)
	wrapped := wordwrap.WrapString(original, w.limit)
	return w.writer.Write([]byte(wrapped))
}


NewPunchCardWriter

// NewPunchCardWriter is a NewWordWrapWriter that limits the line width to 80 columns.
func NewPunchCardWriter(w io.Writer) io.Writer {
	return NewWordWrapWriter(w, 80)
}


NewMaxWidthWriter

// NewMaxWidthWriter is a Writer that supports a limit of characters on every
// line, but doesn't do any word wrapping automatically.
func NewMaxWidthWriter(w io.Writer, maxWidth uint) io.Writer {
	return &maxWidthWriter{
		maxWidth: maxWidth,
		writer:   w,
	}
}


Write

func (m maxWidthWriter) Write(p []byte) (nn int, err error) {
	for _, b := range p {
		if m.currentWidth == m.maxWidth {
			m.writer.Write([]byte{'\n'})
			m.currentWidth = 0
		}
		if b == '\n' {
			m.currentWidth = 0
		}
		_, err := m.writer.Write([]byte{b})
		if err != nil {
			return int(m.written), err
		}
		m.written++
		m.currentWidth++
	}
	return len(p), nil
}


Umask

// Umask is a wrapper for `unix.Umask()` on non-Windows platforms
func Umask(mask int) (old int, err error) {
	return unix.Umask(mask), nil
}


// Umask returns an error on Windows
func Umask(mask int) (int, error) {
	return 0, errors.New("platform and architecture is not supported")
}


// ParseRFC3339 parses an RFC3339 date in either RFC3339Nano or RFC3339 format.
func ParseRFC3339(s string, nowFn func() metav1.Time) (metav1.Time, error) {
	if t, timeErr := time.Parse(time.RFC3339Nano, s); timeErr == nil {
		return metav1.Time{Time: t}, nil
	}
	t, err := time.Parse(time.RFC3339, s)
	if err != nil {
		return metav1.Time{}, err
	}
	return metav1.Time{Time: t}, nil
}


HashObject

// HashObject returns the hash of a Object hash by a Codec
func HashObject(obj runtime.Object, codec runtime.Codec) (string, error) {
	data, err := runtime.Encode(codec, obj)
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("%x", md5.Sum(data)), nil
}


ParseFileSource

// ParseFileSource parses the source given.
//
//	Acceptable formats include:
//	 1.  source-path: the basename will become the key name
//	 2.  source-name=source-path: the source-name will become the key name and
//	     source-path is the path to the key file.
//
// Key names cannot include '='.
func ParseFileSource(source string) (keyName, filePath string, err error) {
	numSeparators := strings.Count(source, "=")
	switch {
	case numSeparators == 0:
		return path.Base(filepath.ToSlash(source)), source, nil
	case numSeparators == 1 && strings.HasPrefix(source, "="):
		return "", "", fmt.Errorf("key name for file path %v missing", strings.TrimPrefix(source, "="))
	case numSeparators == 1 && strings.HasSuffix(source, "="):
		return "", "", fmt.Errorf("file path for key name %v missing", strings.TrimSuffix(source, "="))
	case numSeparators > 1:
		return "", "", errors.New("key names or file paths cannot contain '='")
	default:
		components := strings.Split(source, "=")
		return components[0], components[1], nil
	}
}


ParseLiteralSource

// ParseLiteralSource parses the source key=val pair into its component pieces.
// This functionality is distinguished from strings.SplitN(source, "=", 2) since
// it returns an error in the case of empty keys, values, or a missing equals sign.
func ParseLiteralSource(source string) (keyName, value string, err error) {
	// leading equal is invalid
	if strings.Index(source, "=") == 0 {
		return "", "", fmt.Errorf("invalid literal source %v, expected key=value", source)
	}
	// split after the first equal (so values can have the = character)
	items := strings.SplitN(source, "=", 2)
	if len(items) != 2 {
		return "", "", fmt.Errorf("invalid literal source %v, expected key=value", source)
	}

	return items[0], items[1], nil
}


ValidateBytes

// ValidateBytes never fails for NullSchema.
func (NullSchema) ValidateBytes(data []byte) error { return nil }


// ValidateBytes validates bytes.
func (NoDoubleKeySchema) ValidateBytes(data []byte) error {
	var list []error
	if err := validateNoDuplicateKeys(data, "metadata", "labels"); err != nil {
		list = append(list, err)
	}
	if err := validateNoDuplicateKeys(data, "metadata", "annotations"); err != nil {
		list = append(list, err)
	}
	return utilerrors.NewAggregate(list)
}


validateNoDuplicateKeys

func validateNoDuplicateKeys(data []byte, path ...string) error {
	r := ejson.NewDecoder(bytes.NewReader(data))
	// This is Go being unfriendly. The 'path ...string' comes in as a
	// []string, and SeekTo takes ...interface{}, so we can't just pass
	// the path straight in, we have to copy it.  *sigh*
	ifacePath := []interface{}{}
	for ix := range path {
		ifacePath = append(ifacePath, path[ix])
	}
	found, err := r.SeekTo(ifacePath...)
	if err != nil {
		return err
	}
	if !found {
		return nil
	}
	seen := map[string]bool{}
	for {
		tok, err := r.Token()
		if err != nil {
			return err
		}
		switch t := tok.(type) {
		case json.Delim:
			if t.String() == "}" {
				return nil
			}
		case ejson.KeyString:
			if seen[string(t)] {
				return fmt.Errorf("duplicate key: %s", string(t))
			}
			seen[string(t)] = true
		}
	}
}


ValidateBytes

// ValidateBytes validates bytes per a ConjunctiveSchema.
func (c ConjunctiveSchema) ValidateBytes(data []byte) error {
	var list []error
	schemas := []Schema(c)
	for ix := range schemas {
		if err := schemas[ix].ValidateBytes(data); err != nil {
			list = append(list, err)
		}
	}
	return utilerrors.NewAggregate(list)
}


NewParamVerifyingSchema

func NewParamVerifyingSchema(s Schema, verifier resource.Verifier, directive string) Schema {
	return &paramVerifyingSchema{
		schema:    s,
		verifier:  verifier,
		directive: directive,
	}
}


ValidateBytes

// ValidateBytes validates bytes per a ParamVerifyingSchema
func (c *paramVerifyingSchema) ValidateBytes(data []byte) error {
	obj, err := parse(data)
	if err != nil {
		return err
	}

	gvk, errs := getObjectKind(obj)
	if errs != nil {
		return utilerrors.NewAggregate(errs)
	}

	err = c.verifier.HasSupport(gvk)
	if resource.IsParamUnsupportedError(err) {
		switch c.directive {
		case metav1.FieldValidationStrict:
			return c.schema.ValidateBytes(data)
		case metav1.FieldValidationWarn:
			klog.Warningf("cannot perform warn validation if server-side field validation is unsupported, skipping validation")
		default:
			// can't be reached
			klog.Warningf("unexpected field validation directive: %s, skipping validation", c.directive)
		}
		return nil
	}
	return err
}


NewSchemaValidation

// NewSchemaValidation creates a new Schema that can be used
// to validate objects.
func NewSchemaValidation(resources openapi.Resources) Schema {
	return &schemaValidation{
		resources: resources,
	}
}


ValidateBytes

// ValidateBytes will validates the object against using the Resources
// object.
func (v *schemaValidation) ValidateBytes(data []byte) error {
	obj, err := parse(data)
	if err != nil {
		return err
	}

	gvk, errs := getObjectKind(obj)
	if errs != nil {
		return utilerrors.NewAggregate(errs)
	}

	if (gvk == schema.GroupVersionKind{Version: "v1", Kind: "List"}) {
		return utilerrors.NewAggregate(v.validateList(obj))
	}

	return utilerrors.NewAggregate(v.validateResource(obj, gvk))
}


validateList

func (v *schemaValidation) validateList(object interface{}) []error {
	fields, ok := object.(map[string]interface{})
	if !ok || fields == nil {
		return []error{errors.New("invalid object to validate")}
	}

	allErrors := []error{}
	if _, ok := fields["items"].([]interface{}); !ok {
		return []error{errors.New("invalid object to validate")}
	}
	for _, item := range fields["items"].([]interface{}) {
		if gvk, errs := getObjectKind(item); errs != nil {
			allErrors = append(allErrors, errs...)
		} else {
			allErrors = append(allErrors, v.validateResource(item, gvk)...)
		}
	}
	return allErrors
}


validateResource

func (v *schemaValidation) validateResource(obj interface{}, gvk schema.GroupVersionKind) []error {
	resource := v.resources.LookupResource(gvk)
	if resource == nil {
		// resource is not present, let's just skip validation.
		return nil
	}

	return validation.ValidateModel(obj, resource, gvk.Kind)
}


parse

func parse(data []byte) (interface{}, error) {
	var obj interface{}
	out, err := yaml.ToJSON(data)
	if err != nil {
		return nil, err
	}
	if err := json.Unmarshal(out, &obj); err != nil {
		return nil, err
	}
	return obj, nil
}


getObjectKind

func getObjectKind(object interface{}) (schema.GroupVersionKind, []error) {
	var listErrors []error
	fields, ok := object.(map[string]interface{})
	if !ok || fields == nil {
		listErrors = append(listErrors, errors.New("invalid object to validate"))
		return schema.GroupVersionKind{}, listErrors
	}

	var group string
	var version string
	apiVersion := fields["apiVersion"]
	if apiVersion == nil {
		listErrors = append(listErrors, errors.New("apiVersion not set"))
	} else if _, ok := apiVersion.(string); !ok {
		listErrors = append(listErrors, errors.New("apiVersion isn't string type"))
	} else {
		gv, err := schema.ParseGroupVersion(apiVersion.(string))
		if err != nil {
			listErrors = append(listErrors, err)
		} else {
			group = gv.Group
			version = gv.Version
		}
	}
	kind := fields["kind"]
	if kind == nil {
		listErrors = append(listErrors, errors.New("kind not set"))
	} else if _, ok := kind.(string); !ok {
		listErrors = append(listErrors, errors.New("kind isn't string type"))
	}
	if listErrors != nil {
		return schema.GroupVersionKind{}, listErrors
	}

	return schema.GroupVersionKind{Group: group, Version: version, Kind: kind.(string)}, nil
}


addKnownTypes

// addKnownTypes registers known types to the given scheme
func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&CredentialProviderConfig{},
	)
	return nil
}


// addKnownTypes registers known types to the given scheme
func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&KubeletConfiguration{},
		&SerializedNodeConfigSource{},
		&CredentialProviderConfig{},
	)
	return nil
}


Install

// Install registers the credentialprovider.kubelet.k8s.io APIs into the given scheme.
func Install(scheme *runtime.Scheme) {
	utilruntime.Must(credentialprovider.AddToScheme(scheme))
	utilruntime.Must(v1alpha1.AddToScheme(scheme))
	utilruntime.Must(scheme.SetVersionPriority(v1alpha1.SchemeGroupVersion))
	utilruntime.Must(v1beta1.AddToScheme(scheme))
	utilruntime.Must(scheme.SetVersionPriority(v1beta1.SchemeGroupVersion))
	utilruntime.Must(v1.AddToScheme(scheme))
	utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion))
}


addKnownTypes

func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&CredentialProviderRequest{},
		&CredentialProviderResponse{},
	)
	return nil
}


func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&CredentialProviderRequest{},
		&CredentialProviderResponse{},
	)
	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
	return nil
}


KubeletLabels

// KubeletLabels returns the list of label keys kubelets are allowed to set on their own Node objects
func KubeletLabels() []string {
	return kubeletLabels.List()
}


KubeletLabelNamespaces

// KubeletLabelNamespaces returns the list of label key namespaces kubelets are allowed to set on their own Node objects
func KubeletLabelNamespaces() []string {
	return kubeletLabelNamespaces.List()
}


IsKubeletLabel

// IsKubeletLabel returns true if the label key is one that kubelets are allowed to set on their own Node object.
// This checks if the key is in the KubeletLabels() list, or has a namespace in the KubeletLabelNamespaces() list.
func IsKubeletLabel(key string) bool {
	if kubeletLabels.Has(key) {
		return true
	}

	namespace := getLabelNamespace(key)
	for allowedNamespace := range kubeletLabelNamespaces {
		if namespace == allowedNamespace || strings.HasSuffix(namespace, "."+allowedNamespace) {
			return true
		}
	}

	return false
}


getLabelNamespace

func getLabelNamespace(key string) string {
	if parts := strings.SplitN(key, "/", 2); len(parts) == 2 {
		return parts[0]
	}
	return ""
}


GetServicePrincipalToken

// GetServicePrincipalToken creates a new service principal token based on the configuration.
//
// By default, the cluster and its network resources are deployed in the same AAD Tenant and Subscription,
// and all azure clients use this method to fetch Service Principal Token.
//
// If NetworkResourceTenantID and NetworkResourceSubscriptionID are specified to have different values than TenantID and SubscriptionID, network resources are deployed in different AAD Tenant and Subscription than those for the cluster,
// than only azure clients except VM/VMSS and network resource ones use this method to fetch Token.
// For tokens for VM/VMSS and network resource ones, please check GetMultiTenantServicePrincipalToken and GetNetworkResourceServicePrincipalToken.
func GetServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) (*adal.ServicePrincipalToken, error) {
	var tenantID string
	if strings.EqualFold(config.IdentitySystem, ADFSIdentitySystem) {
		tenantID = ADFSIdentitySystem
	} else {
		tenantID = config.TenantID
	}

	if config.UseManagedIdentityExtension {
		klog.V(2).Infoln("azure: using managed identity extension to retrieve access token")
		msiEndpoint, err := adal.GetMSIVMEndpoint()
		if err != nil {
			return nil, fmt.Errorf("Getting the managed service identity endpoint: %v", err)
		}
		if len(config.UserAssignedIdentityID) > 0 {
			klog.V(4).Info("azure: using User Assigned MSI ID to retrieve access token")
			return adal.NewServicePrincipalTokenFromMSIWithUserAssignedID(msiEndpoint,
				env.ServiceManagementEndpoint,
				config.UserAssignedIdentityID)
		}
		klog.V(4).Info("azure: using System Assigned MSI to retrieve access token")
		return adal.NewServicePrincipalTokenFromMSI(
			msiEndpoint,
			env.ServiceManagementEndpoint)
	}

	oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, tenantID, nil)
	if err != nil {
		return nil, fmt.Errorf("creating the OAuth config: %v", err)
	}

	if len(config.AADClientSecret) > 0 {
		klog.V(2).Infoln("azure: using client_id+client_secret to retrieve access token")
		return adal.NewServicePrincipalToken(
			*oauthConfig,
			config.AADClientID,
			config.AADClientSecret,
			env.ServiceManagementEndpoint)
	}

	if len(config.AADClientCertPath) > 0 && len(config.AADClientCertPassword) > 0 {
		klog.V(2).Infoln("azure: using jwt client_assertion (client_cert+client_private_key) to retrieve access token")
		certData, err := ioutil.ReadFile(config.AADClientCertPath)
		if err != nil {
			return nil, fmt.Errorf("reading the client certificate from file %s: %v", config.AADClientCertPath, err)
		}
		certificate, privateKey, err := decodePkcs12(certData, config.AADClientCertPassword)
		if err != nil {
			return nil, fmt.Errorf("decoding the client certificate: %v", err)
		}
		return adal.NewServicePrincipalTokenFromCertificate(
			*oauthConfig,
			config.AADClientID,
			certificate,
			privateKey,
			env.ServiceManagementEndpoint)
	}

	return nil, ErrorNoAuth
}


GetMultiTenantServicePrincipalToken

// GetMultiTenantServicePrincipalToken is used when (and only when) NetworkResourceTenantID and NetworkResourceSubscriptionID are specified to have different values than TenantID and SubscriptionID.
//
// In that scenario, network resources are deployed in different AAD Tenant and Subscription than those for the cluster,
// and this method creates a new multi-tenant service principal token based on the configuration.
//
// PrimaryToken of the returned multi-tenant token is for the AAD Tenant specified by TenantID, and AuxiliaryToken of the returned multi-tenant token is for the AAD Tenant specified by NetworkResourceTenantID.
//
// Azure VM/VMSS clients use this multi-tenant token, in order to operate those VM/VMSS in AAD Tenant specified by TenantID, and meanwhile in their payload they are referencing network resources (e.g. Load Balancer, Network Security Group, etc.) in AAD Tenant specified by NetworkResourceTenantID.
func GetMultiTenantServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) (*adal.MultiTenantServicePrincipalToken, error) {
	err := config.checkConfigWhenNetworkResourceInDifferentTenant()
	if err != nil {
		return nil, fmt.Errorf("got error(%v) in getting multi-tenant service principal token", err)
	}

	multiTenantOAuthConfig, err := adal.NewMultiTenantOAuthConfig(
		env.ActiveDirectoryEndpoint, config.TenantID, []string{config.NetworkResourceTenantID}, adal.OAuthOptions{})
	if err != nil {
		return nil, fmt.Errorf("creating the multi-tenant OAuth config: %v", err)
	}

	if len(config.AADClientSecret) > 0 {
		klog.V(2).Infoln("azure: using client_id+client_secret to retrieve multi-tenant access token")
		return adal.NewMultiTenantServicePrincipalToken(
			multiTenantOAuthConfig,
			config.AADClientID,
			config.AADClientSecret,
			env.ServiceManagementEndpoint)
	}

	if len(config.AADClientCertPath) > 0 && len(config.AADClientCertPassword) > 0 {
		return nil, fmt.Errorf("AAD Application client certificate authentication is not supported in getting multi-tenant service principal token")
	}

	return nil, ErrorNoAuth
}


GetNetworkResourceServicePrincipalToken

// GetNetworkResourceServicePrincipalToken is used when (and only when) NetworkResourceTenantID and NetworkResourceSubscriptionID are specified to have different values than TenantID and SubscriptionID.
//
// In that scenario, network resources are deployed in different AAD Tenant and Subscription than those for the cluster,
// and this method creates a new service principal token for network resources tenant based on the configuration.
//
// Azure network resource (Load Balancer, Public IP, Route Table, Network Security Group and their sub level resources) clients use this multi-tenant token, in order to operate resources in AAD Tenant specified by NetworkResourceTenantID.
func GetNetworkResourceServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) (*adal.ServicePrincipalToken, error) {
	err := config.checkConfigWhenNetworkResourceInDifferentTenant()
	if err != nil {
		return nil, fmt.Errorf("got error(%v) in getting network resources service principal token", err)
	}

	oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.NetworkResourceTenantID, nil)
	if err != nil {
		return nil, fmt.Errorf("creating the OAuth config for network resources tenant: %v", err)
	}

	if len(config.AADClientSecret) > 0 {
		klog.V(2).Infoln("azure: using client_id+client_secret to retrieve access token for network resources tenant")
		return adal.NewServicePrincipalToken(
			*oauthConfig,
			config.AADClientID,
			config.AADClientSecret,
			env.ServiceManagementEndpoint)
	}

	if len(config.AADClientCertPath) > 0 && len(config.AADClientCertPassword) > 0 {
		return nil, fmt.Errorf("AAD Application client certificate authentication is not supported in getting network resources service principal token")
	}

	return nil, ErrorNoAuth
}


ParseAzureEnvironment

// ParseAzureEnvironment returns the azure environment.
// If 'resourceManagerEndpoint' is set, the environment is computed by querying the cloud's resource manager endpoint.
// Otherwise, a pre-defined Environment is looked up by name.
func ParseAzureEnvironment(cloudName, resourceManagerEndpoint, identitySystem string) (*azure.Environment, error) {
	var env azure.Environment
	var err error
	if resourceManagerEndpoint != "" {
		klog.V(4).Infof("Loading environment from resource manager endpoint: %s", resourceManagerEndpoint)
		nameOverride := azure.OverrideProperty{Key: azure.EnvironmentName, Value: cloudName}
		env, err = azure.EnvironmentFromURL(resourceManagerEndpoint, nameOverride)
		if err == nil {
			azureStackOverrides(&env, resourceManagerEndpoint, identitySystem)
		}
	} else if cloudName == "" {
		klog.V(4).Info("Using public cloud environment")
		env = azure.PublicCloud
	} else {
		klog.V(4).Infof("Using %s environment", cloudName)
		env, err = azure.EnvironmentFromName(cloudName)
	}
	return &env, err
}


UsesNetworkResourceInDifferentTenant

// UsesNetworkResourceInDifferentTenant determines whether the AzureAuthConfig indicates to use network resources in different AAD Tenant and Subscription than those for the cluster
// Return true only when both NetworkResourceTenantID and NetworkResourceSubscriptionID are specified
// and they are not equals to TenantID and SubscriptionID
func (config *AzureAuthConfig) UsesNetworkResourceInDifferentTenant() bool {
	return len(config.NetworkResourceTenantID) > 0 &&
		len(config.NetworkResourceSubscriptionID) > 0 &&
		!strings.EqualFold(config.NetworkResourceTenantID, config.TenantID) &&
		!strings.EqualFold(config.NetworkResourceSubscriptionID, config.SubscriptionID)
}


// decodePkcs12 decodes a PKCS#12 client certificate by extracting the public certificate and
// the private RSA key
func decodePkcs12(pkcs []byte, password string) (*x509.Certificate, *rsa.PrivateKey, error) {
	privateKey, certificate, err := pkcs12.Decode(pkcs, password)
	if err != nil {
		return nil, nil, fmt.Errorf("decoding the PKCS#12 client certificate: %v", err)
	}
	rsaPrivateKey, isRsaKey := privateKey.(*rsa.PrivateKey)
	if !isRsaKey {
		return nil, nil, fmt.Errorf("PKCS#12 certificate must contain a RSA private key")
	}

	return certificate, rsaPrivateKey, nil
}


azureStackOverrides

// azureStackOverrides ensures that the Environment matches what AKSe currently generates for Azure Stack
func azureStackOverrides(env *azure.Environment, resourceManagerEndpoint, identitySystem string) {
	env.ManagementPortalURL = strings.Replace(resourceManagerEndpoint, "https://management.", "https://portal.", -1)
	env.ServiceManagementEndpoint = env.TokenAudience
	env.ResourceManagerVMDNSSuffix = strings.Replace(resourceManagerEndpoint, "https://management.", "cloudapp.", -1)
	env.ResourceManagerVMDNSSuffix = strings.TrimSuffix(env.ResourceManagerVMDNSSuffix, "/")
	if strings.EqualFold(identitySystem, ADFSIdentitySystem) {
		env.ActiveDirectoryEndpoint = strings.TrimSuffix(env.ActiveDirectoryEndpoint, "/")
		env.ActiveDirectoryEndpoint = strings.TrimSuffix(env.ActiveDirectoryEndpoint, "adfs")
	}
}


// checkConfigWhenNetworkResourceInDifferentTenant checks configuration for the scenario of using network resource in different tenant
func (config *AzureAuthConfig) checkConfigWhenNetworkResourceInDifferentTenant() error {
	if !config.UsesNetworkResourceInDifferentTenant() {
		return fmt.Errorf("NetworkResourceTenantID and NetworkResourceSubscriptionID must be configured")
	}

	if strings.EqualFold(config.IdentitySystem, ADFSIdentitySystem) {
		return fmt.Errorf("ADFS identity system is not supported")
	}

	if config.UseManagedIdentityExtension {
		return fmt.Errorf("managed identity is not supported")
	}

	return nil
}


init

func init() {
	// In go-autorest SDK https://github.com/Azure/go-autorest/blob/master/autorest/sender.go#L258-L287,
	// if ARM returns http.StatusTooManyRequests, the sender doesn't increase the retry attempt count,
	// hence the Azure clients will keep retrying forever until it get a status code other than 429.
	// So we explicitly removes http.StatusTooManyRequests from autorest.StatusCodesForRetry.
	// Refer https://github.com/Azure/go-autorest/issues/398.
	// TODO(feiskyer): Use autorest.SendDecorator to customize the retry policy when new Azure SDK is available.
	statusCodesForRetry := make([]int, 0)
	for _, code := range autorest.StatusCodesForRetry {
		if code != http.StatusTooManyRequests {
			statusCodesForRetry = append(statusCodesForRetry, code)
		}
	}
	autorest.StatusCodesForRetry = statusCodesForRetry

	cloudprovider.RegisterCloudProvider(CloudProviderName, NewCloud)
}


NewCloud

// NewCloud returns a Cloud with initialized clients
func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) {
	az, err := NewCloudWithoutFeatureGates(configReader)
	if err != nil {
		return nil, err
	}
	az.ipv6DualStackEnabled = true

	return az, nil
}


NewCloudWithoutFeatureGates

// NewCloudWithoutFeatureGates returns a Cloud without trying to wire the feature gates.  This is used by the unit tests
// that don't load the actual features being used in the cluster.
func NewCloudWithoutFeatureGates(configReader io.Reader) (*Cloud, error) {
	config, err := parseConfig(configReader)
	if err != nil {
		return nil, err
	}

	az := &Cloud{
		nodeNames:                sets.NewString(),
		nodeZones:                map[string]sets.String{},
		nodeResourceGroups:       map[string]string{},
		unmanagedNodes:           sets.NewString(),
		excludeLoadBalancerNodes: sets.NewString(),
		routeCIDRs:               map[string]string{},
	}

	err = az.InitializeCloudFromConfig(config, false)
	if err != nil {
		return nil, err
	}

	return az, nil
}


InitializeCloudFromConfig

// InitializeCloudFromConfig initializes the Cloud from config.
func (az *Cloud) InitializeCloudFromConfig(config *Config, fromSecret bool) error {
	// cloud-config not set, return nil so that it would be initialized from secret.
	if config == nil {
		klog.Warning("cloud-config is not provided, Azure cloud provider would be initialized from secret")
		return nil
	}

	if config.RouteTableResourceGroup == "" {
		config.RouteTableResourceGroup = config.ResourceGroup
	}

	if config.SecurityGroupResourceGroup == "" {
		config.SecurityGroupResourceGroup = config.ResourceGroup
	}

	if config.VMType == "" {
		// default to standard vmType if not set.
		config.VMType = vmTypeStandard
	}

	if config.DisableAvailabilitySetNodes && config.VMType != vmTypeVMSS {
		return fmt.Errorf("disableAvailabilitySetNodes %v is only supported when vmType is 'vmss'", config.DisableAvailabilitySetNodes)
	}

	if config.CloudConfigType == "" {
		// The default cloud config type is cloudConfigTypeMerge.
		config.CloudConfigType = cloudConfigTypeMerge
	} else {
		supportedCloudConfigTypes := sets.NewString(
			string(cloudConfigTypeMerge),
			string(cloudConfigTypeFile),
			string(cloudConfigTypeSecret))
		if !supportedCloudConfigTypes.Has(string(config.CloudConfigType)) {
			return fmt.Errorf("cloudConfigType %v is not supported, supported values are %v", config.CloudConfigType, supportedCloudConfigTypes.List())
		}
	}

	env, err := auth.ParseAzureEnvironment(config.Cloud, config.ResourceManagerEndpoint, config.IdentitySystem)
	if err != nil {
		return err
	}

	servicePrincipalToken, err := auth.GetServicePrincipalToken(&config.AzureAuthConfig, env)
	if err == auth.ErrorNoAuth {
		// Only controller-manager would lazy-initialize from secret, and credentials are required for such case.
		if fromSecret {
			err := fmt.Errorf("no credentials provided for Azure cloud provider")
			klog.Fatalf("%v", err)
			return err
		}

		// No credentials provided, useInstanceMetadata should be enabled for Kubelet.
		// TODO(feiskyer): print different error message for Kubelet and controller-manager, as they're
		// requiring different credential settings.
		if !config.UseInstanceMetadata && config.CloudConfigType == cloudConfigTypeFile {
			return fmt.Errorf("useInstanceMetadata must be enabled without Azure credentials")
		}

		klog.V(2).Infof("Azure cloud provider is starting without credentials")
	} else if err != nil {
		return err
	}

	// Initialize rate limiting config options.
	InitializeCloudProviderRateLimitConfig(&config.CloudProviderRateLimitConfig)

	// Conditionally configure resource request backoff
	resourceRequestBackoff := wait.Backoff{
		Steps: 1,
	}
	if config.CloudProviderBackoff {
		// Assign backoff defaults if no configuration was passed in
		if config.CloudProviderBackoffRetries == 0 {
			config.CloudProviderBackoffRetries = backoffRetriesDefault
		}
		if config.CloudProviderBackoffDuration == 0 {
			config.CloudProviderBackoffDuration = backoffDurationDefault
		}
		if config.CloudProviderBackoffExponent == 0 {
			config.CloudProviderBackoffExponent = backoffExponentDefault
		}

		if config.CloudProviderBackoffJitter == 0 {
			config.CloudProviderBackoffJitter = backoffJitterDefault
		}

		resourceRequestBackoff = wait.Backoff{
			Steps:    config.CloudProviderBackoffRetries,
			Factor:   config.CloudProviderBackoffExponent,
			Duration: time.Duration(config.CloudProviderBackoffDuration) * time.Second,
			Jitter:   config.CloudProviderBackoffJitter,
		}
		klog.V(2).Infof("Azure cloudprovider using try backoff: retries=%d, exponent=%f, duration=%d, jitter=%f",
			config.CloudProviderBackoffRetries,
			config.CloudProviderBackoffExponent,
			config.CloudProviderBackoffDuration,
			config.CloudProviderBackoffJitter)
	} else {
		// CloudProviderBackoffRetries will be set to 1 by default as the requirements of Azure SDK.
		config.CloudProviderBackoffRetries = 1
		config.CloudProviderBackoffDuration = backoffDurationDefault
	}

	if strings.EqualFold(config.LoadBalancerSku, loadBalancerSkuStandard) {
		// Do not add master nodes to standard LB by default.
		if config.ExcludeMasterFromStandardLB == nil {
			config.ExcludeMasterFromStandardLB = &defaultExcludeMasterFromStandardLB
		}

		// Enable outbound SNAT by default.
		if config.DisableOutboundSNAT == nil {
			config.DisableOutboundSNAT = &defaultDisableOutboundSNAT
		}
	} else {
		if config.DisableOutboundSNAT != nil && *config.DisableOutboundSNAT {
			return fmt.Errorf("disableOutboundSNAT should only set when loadBalancerSku is standard")
		}
	}

	az.Config = *config
	az.Environment = *env
	az.ResourceRequestBackoff = resourceRequestBackoff
	az.metadata, err = NewInstanceMetadataService(imdsServer)
	if err != nil {
		return err
	}

	// No credentials provided, InstanceMetadataService would be used for getting Azure resources.
	// Note that this only applies to Kubelet, controller-manager should configure credentials for managing Azure resources.
	if servicePrincipalToken == nil {
		return nil
	}

	// If uses network resources in different AAD Tenant, then prepare corresponding Service Principal Token for VM/VMSS client and network resources client
	var multiTenantServicePrincipalToken *adal.MultiTenantServicePrincipalToken
	var networkResourceServicePrincipalToken *adal.ServicePrincipalToken
	if az.Config.UsesNetworkResourceInDifferentTenant() {
		multiTenantServicePrincipalToken, err = auth.GetMultiTenantServicePrincipalToken(&az.Config.AzureAuthConfig, &az.Environment)
		if err != nil {
			return err
		}
		networkResourceServicePrincipalToken, err = auth.GetNetworkResourceServicePrincipalToken(&az.Config.AzureAuthConfig, &az.Environment)
		if err != nil {
			return err
		}
	}

	az.configAzureClients(servicePrincipalToken, multiTenantServicePrincipalToken, networkResourceServicePrincipalToken)

	if az.MaximumLoadBalancerRuleCount == 0 {
		az.MaximumLoadBalancerRuleCount = maximumLoadBalancerRuleCount
	}

	if strings.EqualFold(vmTypeVMSS, az.Config.VMType) {
		az.VMSet, err = newScaleSet(az)
		if err != nil {
			return err
		}
	} else {
		az.VMSet = newAvailabilitySet(az)
	}

	az.vmCache, err = az.newVMCache()
	if err != nil {
		return err
	}

	az.lbCache, err = az.newLBCache()
	if err != nil {
		return err
	}

	az.nsgCache, err = az.newNSGCache()
	if err != nil {
		return err
	}

	az.rtCache, err = az.newRouteTableCache()
	if err != nil {
		return err
	}

	if err := initDiskControllers(az); err != nil {
		return err
	}

	// start delayed route updater.
	az.routeUpdater = newDelayedRouteUpdater(az, routeUpdateInterval)
	go az.routeUpdater.run()

	return nil
}


configAzureClients

func (az *Cloud) configAzureClients(
	servicePrincipalToken *adal.ServicePrincipalToken,
	multiTenantServicePrincipalToken *adal.MultiTenantServicePrincipalToken,
	networkResourceServicePrincipalToken *adal.ServicePrincipalToken) {
	azClientConfig := az.getAzureClientConfig(servicePrincipalToken)

	// Prepare AzureClientConfig for all azure clients
	interfaceClientConfig := azClientConfig.WithRateLimiter(az.Config.InterfaceRateLimit)
	vmSizeClientConfig := azClientConfig.WithRateLimiter(az.Config.VirtualMachineSizeRateLimit)
	snapshotClientConfig := azClientConfig.WithRateLimiter(az.Config.SnapshotRateLimit)
	storageAccountClientConfig := azClientConfig.WithRateLimiter(az.Config.StorageAccountRateLimit)
	diskClientConfig := azClientConfig.WithRateLimiter(az.Config.DiskRateLimit)
	vmClientConfig := azClientConfig.WithRateLimiter(az.Config.VirtualMachineRateLimit)
	vmssClientConfig := azClientConfig.WithRateLimiter(az.Config.VirtualMachineScaleSetRateLimit)
	// Error "not an active Virtual Machine Scale Set VM" is not retriable for VMSS VM.
	// But http.StatusNotFound is retriable because of ARM replication latency.
	vmssVMClientConfig := azClientConfig.WithRateLimiter(az.Config.VirtualMachineScaleSetRateLimit)
	vmssVMClientConfig.Backoff = vmssVMClientConfig.Backoff.WithNonRetriableErrors([]string{vmssVMNotActiveErrorMessage}).WithRetriableHTTPStatusCodes([]int{http.StatusNotFound})
	routeClientConfig := azClientConfig.WithRateLimiter(az.Config.RouteRateLimit)
	subnetClientConfig := azClientConfig.WithRateLimiter(az.Config.SubnetsRateLimit)
	routeTableClientConfig := azClientConfig.WithRateLimiter(az.Config.RouteTableRateLimit)
	loadBalancerClientConfig := azClientConfig.WithRateLimiter(az.Config.LoadBalancerRateLimit)
	securityGroupClientConfig := azClientConfig.WithRateLimiter(az.Config.SecurityGroupRateLimit)
	publicIPClientConfig := azClientConfig.WithRateLimiter(az.Config.PublicIPAddressRateLimit)
	// TODO(ZeroMagic): add azurefileRateLimit
	fileClientConfig := azClientConfig.WithRateLimiter(nil)

	// If uses network resources in different AAD Tenant, update Authorizer for VM/VMSS client config
	if multiTenantServicePrincipalToken != nil {
		multiTenantServicePrincipalTokenAuthorizer := autorest.NewMultiTenantServicePrincipalTokenAuthorizer(multiTenantServicePrincipalToken)
		vmClientConfig.Authorizer = multiTenantServicePrincipalTokenAuthorizer
		vmssClientConfig.Authorizer = multiTenantServicePrincipalTokenAuthorizer
		vmssVMClientConfig.Authorizer = multiTenantServicePrincipalTokenAuthorizer
	}

	// If uses network resources in different AAD Tenant, update SubscriptionID and Authorizer for network resources client config
	if networkResourceServicePrincipalToken != nil {
		networkResourceServicePrincipalTokenAuthorizer := autorest.NewBearerAuthorizer(networkResourceServicePrincipalToken)
		routeClientConfig.Authorizer = networkResourceServicePrincipalTokenAuthorizer
		subnetClientConfig.Authorizer = networkResourceServicePrincipalTokenAuthorizer
		routeTableClientConfig.Authorizer = networkResourceServicePrincipalTokenAuthorizer
		loadBalancerClientConfig.Authorizer = networkResourceServicePrincipalTokenAuthorizer
		securityGroupClientConfig.Authorizer = networkResourceServicePrincipalTokenAuthorizer
		publicIPClientConfig.Authorizer = networkResourceServicePrincipalTokenAuthorizer

		routeClientConfig.SubscriptionID = az.Config.NetworkResourceSubscriptionID
		subnetClientConfig.SubscriptionID = az.Config.NetworkResourceSubscriptionID
		routeTableClientConfig.SubscriptionID = az.Config.NetworkResourceSubscriptionID
		loadBalancerClientConfig.SubscriptionID = az.Config.NetworkResourceSubscriptionID
		securityGroupClientConfig.SubscriptionID = az.Config.NetworkResourceSubscriptionID
		publicIPClientConfig.SubscriptionID = az.Config.NetworkResourceSubscriptionID
	}

	// Initialize all azure clients based on client config
	az.InterfacesClient = interfaceclient.New(interfaceClientConfig)
	az.VirtualMachineSizesClient = vmsizeclient.New(vmSizeClientConfig)
	az.SnapshotsClient = snapshotclient.New(snapshotClientConfig)
	az.StorageAccountClient = storageaccountclient.New(storageAccountClientConfig)
	az.DisksClient = diskclient.New(diskClientConfig)
	az.VirtualMachinesClient = vmclient.New(vmClientConfig)
	az.VirtualMachineScaleSetsClient = vmssclient.New(vmssClientConfig)
	az.VirtualMachineScaleSetVMsClient = vmssvmclient.New(vmssVMClientConfig)
	az.RoutesClient = routeclient.New(routeClientConfig)
	az.SubnetsClient = subnetclient.New(subnetClientConfig)
	az.RouteTablesClient = routetableclient.New(routeTableClientConfig)
	az.LoadBalancerClient = loadbalancerclient.New(loadBalancerClientConfig)
	az.SecurityGroupsClient = securitygroupclient.New(securityGroupClientConfig)
	az.PublicIPAddressesClient = publicipclient.New(publicIPClientConfig)
	az.FileClient = fileclient.New(fileClientConfig)
}


getAzureClientConfig

func (az *Cloud) getAzureClientConfig(servicePrincipalToken *adal.ServicePrincipalToken) *azclients.ClientConfig {
	azClientConfig := &azclients.ClientConfig{
		CloudName:               az.Config.Cloud,
		Location:                az.Config.Location,
		SubscriptionID:          az.Config.SubscriptionID,
		ResourceManagerEndpoint: az.Environment.ResourceManagerEndpoint,
		Authorizer:              autorest.NewBearerAuthorizer(servicePrincipalToken),
		Backoff:                 &retry.Backoff{Steps: 1},
	}

	if az.Config.CloudProviderBackoff {
		azClientConfig.Backoff = &retry.Backoff{
			Steps:    az.Config.CloudProviderBackoffRetries,
			Factor:   az.Config.CloudProviderBackoffExponent,
			Duration: time.Duration(az.Config.CloudProviderBackoffDuration) * time.Second,
			Jitter:   az.Config.CloudProviderBackoffJitter,
		}
	}

	return azClientConfig
}


parseConfig

// parseConfig returns a parsed configuration for an Azure cloudprovider config file
func parseConfig(configReader io.Reader) (*Config, error) {
	var config Config
	if configReader == nil {
		return nil, nil
	}

	configContents, err := ioutil.ReadAll(configReader)
	if err != nil {
		return nil, err
	}

	err = yaml.Unmarshal(configContents, &config)
	if err != nil {
		return nil, err
	}

	// The resource group name may be in different cases from different Azure APIs, hence it is converted to lower here.
	// See more context at https://github.com/kubernetes/kubernetes/issues/71994.
	config.ResourceGroup = strings.ToLower(config.ResourceGroup)
	return &config, nil
}


Initialize

// Initialize passes a Kubernetes clientBuilder interface to the cloud provider
func (az *Cloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, stop <-chan struct{}) {
	az.KubeClient = clientBuilder.ClientOrDie("azure-cloud-provider")
	az.eventBroadcaster = record.NewBroadcaster()
	az.eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: az.KubeClient.CoreV1().Events("")})
	az.eventRecorder = az.eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "azure-cloud-provider"})
	az.InitializeCloudFromSecret()
}


LoadBalancer

// LoadBalancer returns a balancer interface. Also returns true if the interface is supported, false otherwise.
func (az *Cloud) LoadBalancer() (cloudprovider.LoadBalancer, bool) {
	return az, true
}


Instances

// Instances returns an instances interface. Also returns true if the interface is supported, false otherwise.
func (az *Cloud) Instances() (cloudprovider.Instances, bool) {
	return az, true
}


// InstancesV2 returns an instancesV2 interface. Also returns true if the interface is supported, false otherwise.
// TODO: implement ONLY for external cloud provider
func (az *Cloud) InstancesV2() (cloudprovider.InstancesV2, bool) {
	return nil, false
}


Zones

// Zones returns a zones interface. Also returns true if the interface is supported, false otherwise.
func (az *Cloud) Zones() (cloudprovider.Zones, bool) {
	return az, true
}


Clusters

// Clusters returns a clusters interface.  Also returns true if the interface is supported, false otherwise.
func (az *Cloud) Clusters() (cloudprovider.Clusters, bool) {
	return nil, false
}


Routes

// Routes returns a routes interface along with whether the interface is supported.
func (az *Cloud) Routes() (cloudprovider.Routes, bool) {
	return az, true
}


HasClusterID

// HasClusterID returns true if the cluster has a clusterID
func (az *Cloud) HasClusterID() bool {
	return true
}


ProviderName

// ProviderName returns the cloud provider ID.
func (az *Cloud) ProviderName() string {
	return CloudProviderName
}


initDiskControllers

func initDiskControllers(az *Cloud) error {
	// Common controller contains the function
	// needed by both blob disk and managed disk controllers

	common := &controllerCommon{
		location:              az.Location,
		storageEndpointSuffix: az.Environment.StorageEndpointSuffix,
		resourceGroup:         az.ResourceGroup,
		subscriptionID:        az.SubscriptionID,
		cloud:                 az,
		vmLockMap:             newLockMap(),
	}

	az.BlobDiskController = &BlobDiskController{common: common}
	az.ManagedDiskController = &ManagedDiskController{common: common}
	az.controllerCommon = common

	return nil
}


SetInformers

// SetInformers sets informers for Azure cloud provider.
func (az *Cloud) SetInformers(informerFactory informers.SharedInformerFactory) {
	klog.Infof("Setting up informers for Azure cloud provider")
	nodeInformer := informerFactory.Core().V1().Nodes().Informer()
	nodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			node := obj.(*v1.Node)
			az.updateNodeCaches(nil, node)
		},
		UpdateFunc: func(prev, obj interface{}) {
			prevNode := prev.(*v1.Node)
			newNode := obj.(*v1.Node)
			az.updateNodeCaches(prevNode, newNode)
		},
		DeleteFunc: func(obj interface{}) {
			node, isNode := obj.(*v1.Node)
			// We can get DeletedFinalStateUnknown instead of *v1.Node here
			// and we need to handle that correctly.
			if !isNode {
				deletedState, ok := obj.(cache.DeletedFinalStateUnknown)
				if !ok {
					klog.Errorf("Received unexpected object: %v", obj)
					return
				}
				node, ok = deletedState.Obj.(*v1.Node)
				if !ok {
					klog.Errorf("DeletedFinalStateUnknown contained non-Node object: %v", deletedState.Obj)
					return
				}
			}
			az.updateNodeCaches(node, nil)
		},
	})
	az.nodeInformerSynced = nodeInformer.HasSynced
}


updateNodeCaches

// updateNodeCaches updates local cache for node's zones and external resource groups.
func (az *Cloud) updateNodeCaches(prevNode, newNode *v1.Node) {
	az.nodeCachesLock.Lock()
	defer az.nodeCachesLock.Unlock()

	if prevNode != nil {

		// Remove from nodeNames cache.
		az.nodeNames.Delete(prevNode.ObjectMeta.Name)

		// Remove from nodeZones cache
		prevZone, ok := prevNode.ObjectMeta.Labels[v1.LabelTopologyZone]

		if ok && az.isAvailabilityZone(prevZone) {
			az.nodeZones[prevZone].Delete(prevNode.ObjectMeta.Name)
			if az.nodeZones[prevZone].Len() == 0 {
				az.nodeZones[prevZone] = nil
			}
		}

		// Remove from nodeZones cache if using deprecated LabelFailureDomainBetaZone
		prevZoneFailureDomain, ok := prevNode.ObjectMeta.Labels[v1.LabelFailureDomainBetaZone]
		if ok && az.isAvailabilityZone(prevZoneFailureDomain) {
			az.nodeZones[prevZone].Delete(prevNode.ObjectMeta.Name)
			if az.nodeZones[prevZone].Len() == 0 {
				az.nodeZones[prevZone] = nil
			}
		}

		// Remove from nodeResourceGroups cache.
		_, ok = prevNode.ObjectMeta.Labels[externalResourceGroupLabel]
		if ok {
			delete(az.nodeResourceGroups, prevNode.ObjectMeta.Name)
		}

		managed, ok := prevNode.ObjectMeta.Labels[managedByAzureLabel]
		isNodeManagedByCloudProvider := !ok || managed != "false"

		// Remove from unmanagedNodes cache
		if !isNodeManagedByCloudProvider {
			az.unmanagedNodes.Delete(prevNode.ObjectMeta.Name)
		}

		// if the node is being deleted from the cluster, exclude it from load balancers
		if newNode == nil {
			az.excludeLoadBalancerNodes.Insert(prevNode.ObjectMeta.Name)
		}
	}

	if newNode != nil {
		// Add to nodeNames cache.
		az.nodeNames.Insert(newNode.ObjectMeta.Name)

		// Add to nodeZones cache.
		newZone, ok := newNode.ObjectMeta.Labels[v1.LabelTopologyZone]
		if ok && az.isAvailabilityZone(newZone) {
			if az.nodeZones[newZone] == nil {
				az.nodeZones[newZone] = sets.NewString()
			}
			az.nodeZones[newZone].Insert(newNode.ObjectMeta.Name)
		}

		// Add to nodeResourceGroups cache.
		newRG, ok := newNode.ObjectMeta.Labels[externalResourceGroupLabel]
		if ok && len(newRG) > 0 {
			az.nodeResourceGroups[newNode.ObjectMeta.Name] = strings.ToLower(newRG)
		}

		_, hasExcludeBalancerLabel := newNode.ObjectMeta.Labels[v1.LabelNodeExcludeBalancers]
		managed, ok := newNode.ObjectMeta.Labels[managedByAzureLabel]
		isNodeManagedByCloudProvider := !ok || managed != "false"

		// Update unmanagedNodes cache
		if !isNodeManagedByCloudProvider {
			az.unmanagedNodes.Insert(newNode.ObjectMeta.Name)
		}

		// Update excludeLoadBalancerNodes cache
		switch {
		case !isNodeManagedByCloudProvider:
			az.excludeLoadBalancerNodes.Insert(newNode.ObjectMeta.Name)

		case hasExcludeBalancerLabel:
			az.excludeLoadBalancerNodes.Insert(newNode.ObjectMeta.Name)

		case !isNodeReady(newNode) && getCloudTaint(newNode.Spec.Taints) == nil:
			// If not in ready state and not a newly created node, add to excludeLoadBalancerNodes cache.
			// New nodes (tainted with "node.cloudprovider.kubernetes.io/uninitialized") should not be
			// excluded from load balancers regardless of their state, so as to reduce the number of
			// VMSS API calls and not provoke VMScaleSetActiveModelsCountLimitReached.
			// (https://github.com/kubernetes-sigs/cloud-provider-azure/issues/851)
			az.excludeLoadBalancerNodes.Insert(newNode.ObjectMeta.Name)

		default:
			// Nodes not falling into the three cases above are valid backends and
			// should not appear in excludeLoadBalancerNodes cache.
			az.excludeLoadBalancerNodes.Delete(newNode.ObjectMeta.Name)
		}
	}
}


GetActiveZones

// GetActiveZones returns all the zones in which k8s nodes are currently running.
func (az *Cloud) GetActiveZones() (sets.String, error) {
	if az.nodeInformerSynced == nil {
		return nil, fmt.Errorf("azure cloud provider doesn't have informers set")
	}

	az.nodeCachesLock.RLock()
	defer az.nodeCachesLock.RUnlock()
	if !az.nodeInformerSynced() {
		return nil, fmt.Errorf("node informer is not synced when trying to GetActiveZones")
	}

	zones := sets.NewString()
	for zone, nodes := range az.nodeZones {
		if len(nodes) > 0 {
			zones.Insert(zone)
		}
	}
	return zones, nil
}


GetLocation

// GetLocation returns the location in which k8s cluster is currently running.
func (az *Cloud) GetLocation() string {
	return az.Location
}


GetNodeResourceGroup

// GetNodeResourceGroup gets resource group for given node.
func (az *Cloud) GetNodeResourceGroup(nodeName string) (string, error) {
	// Kubelet won't set az.nodeInformerSynced, always return configured resourceGroup.
	if az.nodeInformerSynced == nil {
		return az.ResourceGroup, nil
	}

	az.nodeCachesLock.RLock()
	defer az.nodeCachesLock.RUnlock()
	if !az.nodeInformerSynced() {
		return "", fmt.Errorf("node informer is not synced when trying to GetNodeResourceGroup")
	}

	// Return external resource group if it has been cached.
	if cachedRG, ok := az.nodeResourceGroups[nodeName]; ok {
		return cachedRG, nil
	}

	// Return resource group from cloud provider options.
	return az.ResourceGroup, nil
}


GetNodeNames

// GetNodeNames returns a set of all node names in the k8s cluster.
func (az *Cloud) GetNodeNames() (sets.String, error) {
	// Kubelet won't set az.nodeInformerSynced, return nil.
	if az.nodeInformerSynced == nil {
		return nil, nil
	}

	az.nodeCachesLock.RLock()
	defer az.nodeCachesLock.RUnlock()
	if !az.nodeInformerSynced() {
		return nil, fmt.Errorf("node informer is not synced when trying to GetNodeNames")
	}

	return sets.NewString(az.nodeNames.List()...), nil
}


GetResourceGroups

// GetResourceGroups returns a set of resource groups that all nodes are running on.
func (az *Cloud) GetResourceGroups() (sets.String, error) {
	// Kubelet won't set az.nodeInformerSynced, always return configured resourceGroup.
	if az.nodeInformerSynced == nil {
		return sets.NewString(az.ResourceGroup), nil
	}

	az.nodeCachesLock.RLock()
	defer az.nodeCachesLock.RUnlock()
	if !az.nodeInformerSynced() {
		return nil, fmt.Errorf("node informer is not synced when trying to GetResourceGroups")
	}

	resourceGroups := sets.NewString(az.ResourceGroup)
	for _, rg := range az.nodeResourceGroups {
		resourceGroups.Insert(rg)
	}

	return resourceGroups, nil
}


GetUnmanagedNodes

// GetUnmanagedNodes returns a list of nodes not managed by Azure cloud provider (e.g. on-prem nodes).
func (az *Cloud) GetUnmanagedNodes() (sets.String, error) {
	// Kubelet won't set az.nodeInformerSynced, always return nil.
	if az.nodeInformerSynced == nil {
		return nil, nil
	}

	az.nodeCachesLock.RLock()
	defer az.nodeCachesLock.RUnlock()
	if !az.nodeInformerSynced() {
		return nil, fmt.Errorf("node informer is not synced when trying to GetUnmanagedNodes")
	}

	return sets.NewString(az.unmanagedNodes.List()...), nil
}


ShouldNodeExcludedFromLoadBalancer

// ShouldNodeExcludedFromLoadBalancer returns true if node is unmanaged, in external resource group or labeled with "node.kubernetes.io/exclude-from-external-load-balancers".
func (az *Cloud) ShouldNodeExcludedFromLoadBalancer(nodeName string) (bool, error) {
	// Kubelet won't set az.nodeInformerSynced, always return nil.
	if az.nodeInformerSynced == nil {
		return false, nil
	}

	az.nodeCachesLock.RLock()
	defer az.nodeCachesLock.RUnlock()
	if !az.nodeInformerSynced() {
		return false, fmt.Errorf("node informer is not synced when trying to fetch node caches")
	}

	// Return true if the node is in external resource group.
	if cachedRG, ok := az.nodeResourceGroups[nodeName]; ok && !strings.EqualFold(cachedRG, az.ResourceGroup) {
		return true, nil
	}

	return az.excludeLoadBalancerNodes.Has(nodeName), nil
}


isNodeReady

func isNodeReady(node *v1.Node) bool {
	for _, cond := range node.Status.Conditions {
		if cond.Type == v1.NodeReady && cond.Status == v1.ConditionTrue {
			return true
		}
	}
	return false
}


getCloudTaint

func getCloudTaint(taints []v1.Taint) *v1.Taint {
	for _, taint := range taints {
		if taint.Key == cloudproviderapi.TaintExternalCloudProvider {
			return &taint
		}
	}
	return nil
}


RequestBackoff

// RequestBackoff if backoff is disabled in cloud provider it
// returns a new Backoff object steps = 1
// This is to make sure that the requested command executes
// at least once
func (az *Cloud) RequestBackoff() (resourceRequestBackoff wait.Backoff) {
	if az.CloudProviderBackoff {
		return az.ResourceRequestBackoff
	}
	resourceRequestBackoff = wait.Backoff{
		Steps: 1,
	}
	return resourceRequestBackoff
}


Event

// Event creates a event for the specified object.
func (az *Cloud) Event(obj runtime.Object, eventType, reason, message string) {
	if obj != nil && reason != "" {
		az.eventRecorder.Event(obj, eventType, reason, message)
	}
}


GetVirtualMachineWithRetry

// GetVirtualMachineWithRetry invokes az.getVirtualMachine with exponential backoff retry
func (az *Cloud) GetVirtualMachineWithRetry(name types.NodeName, crt azcache.AzureCacheReadType) (compute.VirtualMachine, error) {
	var machine compute.VirtualMachine
	var retryErr error
	err := wait.ExponentialBackoff(az.RequestBackoff(), func() (bool, error) {
		machine, retryErr = az.getVirtualMachine(name, crt)
		if retryErr == cloudprovider.InstanceNotFound {
			return true, cloudprovider.InstanceNotFound
		}
		if retryErr != nil {
			klog.Errorf("GetVirtualMachineWithRetry(%s): backoff failure, will retry, err=%v", name, retryErr)
			return false, nil
		}
		klog.V(2).Infof("GetVirtualMachineWithRetry(%s): backoff success", name)
		return true, nil
	})
	if err == wait.ErrWaitTimeout {
		err = retryErr
	}
	return machine, err
}


ListVirtualMachines

// ListVirtualMachines invokes az.VirtualMachinesClient.List with exponential backoff retry
func (az *Cloud) ListVirtualMachines(resourceGroup string) ([]compute.VirtualMachine, error) {
	ctx, cancel := getContextWithCancel()
	defer cancel()

	allNodes, rerr := az.VirtualMachinesClient.List(ctx, resourceGroup)
	if rerr != nil {
		klog.Errorf("VirtualMachinesClient.List(%v) failure with err=%v", resourceGroup, rerr)
		return nil, rerr.Error()
	}
	klog.V(2).Infof("VirtualMachinesClient.List(%v) success", resourceGroup)
	return allNodes, nil
}


getPrivateIPsForMachine

// getPrivateIPsForMachine is wrapper for optional backoff getting private ips
// list of a node by name
func (az *Cloud) getPrivateIPsForMachine(nodeName types.NodeName) ([]string, error) {
	return az.getPrivateIPsForMachineWithRetry(nodeName)
}


getPrivateIPsForMachineWithRetry

func (az *Cloud) getPrivateIPsForMachineWithRetry(nodeName types.NodeName) ([]string, error) {
	var privateIPs []string
	err := wait.ExponentialBackoff(az.RequestBackoff(), func() (bool, error) {
		var retryErr error
		privateIPs, retryErr = az.VMSet.GetPrivateIPsByNodeName(string(nodeName))
		if retryErr != nil {
			// won't retry since the instance doesn't exist on Azure.
			if retryErr == cloudprovider.InstanceNotFound {
				return true, retryErr
			}
			klog.Errorf("GetPrivateIPsByNodeName(%s): backoff failure, will retry,err=%v", nodeName, retryErr)
			return false, nil
		}
		klog.V(3).Infof("GetPrivateIPsByNodeName(%s): backoff success", nodeName)
		return true, nil
	})
	return privateIPs, err
}


getIPForMachine

func (az *Cloud) getIPForMachine(nodeName types.NodeName) (string, string, error) {
	return az.GetIPForMachineWithRetry(nodeName)
}


GetIPForMachineWithRetry

// GetIPForMachineWithRetry invokes az.getIPForMachine with exponential backoff retry
func (az *Cloud) GetIPForMachineWithRetry(name types.NodeName) (string, string, error) {
	var ip, publicIP string
	err := wait.ExponentialBackoff(az.RequestBackoff(), func() (bool, error) {
		var retryErr error
		ip, publicIP, retryErr = az.VMSet.GetIPByNodeName(string(name))
		if retryErr != nil {
			klog.Errorf("GetIPForMachineWithRetry(%s): backoff failure, will retry,err=%v", name, retryErr)
			return false, nil
		}
		klog.V(3).Infof("GetIPForMachineWithRetry(%s): backoff success", name)
		return true, nil
	})
	return ip, publicIP, err
}


CreateOrUpdateSecurityGroup

// CreateOrUpdateSecurityGroup invokes az.SecurityGroupsClient.CreateOrUpdate with exponential backoff retry
func (az *Cloud) CreateOrUpdateSecurityGroup(sg network.SecurityGroup) error {
	ctx, cancel := getContextWithCancel()
	defer cancel()

	rerr := az.SecurityGroupsClient.CreateOrUpdate(ctx, az.SecurityGroupResourceGroup, *sg.Name, sg, pointer.StringDeref(sg.Etag, ""))
	klog.V(10).Infof("SecurityGroupsClient.CreateOrUpdate(%s): end", *sg.Name)
	if rerr == nil {
		// Invalidate the cache right after updating
		az.nsgCache.Delete(*sg.Name)
		return nil
	}

	// Invalidate the cache because ETAG precondition mismatch.
	if rerr.HTTPStatusCode == http.StatusPreconditionFailed {
		klog.V(3).Infof("SecurityGroup cache for %s is cleanup because of http.StatusPreconditionFailed", *sg.Name)
		az.nsgCache.Delete(*sg.Name)
	}

	// Invalidate the cache because another new operation has canceled the current request.
	if strings.Contains(strings.ToLower(rerr.Error().Error()), operationCanceledErrorMessage) {
		klog.V(3).Infof("SecurityGroup cache for %s is cleanup because CreateOrUpdateSecurityGroup is canceled by another operation", *sg.Name)
		az.nsgCache.Delete(*sg.Name)
	}

	return rerr.Error()
}


cleanupSubnetInFrontendIPConfigurations

func cleanupSubnetInFrontendIPConfigurations(lb *network.LoadBalancer) network.LoadBalancer {
	if lb.LoadBalancerPropertiesFormat == nil || lb.FrontendIPConfigurations == nil {
		return *lb
	}

	frontendIPConfigurations := *lb.FrontendIPConfigurations
	for i := range frontendIPConfigurations {
		config := frontendIPConfigurations[i]
		if config.FrontendIPConfigurationPropertiesFormat != nil &&
			config.Subnet != nil &&
			config.Subnet.ID != nil {
			subnet := network.Subnet{
				ID: config.Subnet.ID,
			}
			if config.Subnet.Name != nil {
				subnet.Name = config.FrontendIPConfigurationPropertiesFormat.Subnet.Name
			}
			config.FrontendIPConfigurationPropertiesFormat.Subnet = &subnet
			frontendIPConfigurations[i] = config
			continue
		}
	}

	lb.FrontendIPConfigurations = &frontendIPConfigurations
	return *lb
}


CreateOrUpdateLB

// CreateOrUpdateLB invokes az.LoadBalancerClient.CreateOrUpdate with exponential backoff retry
func (az *Cloud) CreateOrUpdateLB(service *v1.Service, lb network.LoadBalancer) error {
	ctx, cancel := getContextWithCancel()
	defer cancel()

	lb = cleanupSubnetInFrontendIPConfigurations(&lb)

	rgName := az.getLoadBalancerResourceGroup()
	rerr := az.LoadBalancerClient.CreateOrUpdate(ctx, rgName, pointer.StringDeref(lb.Name, ""), lb, pointer.StringDeref(lb.Etag, ""))
	klog.V(10).Infof("LoadBalancerClient.CreateOrUpdate(%s): end", *lb.Name)
	if rerr == nil {
		// Invalidate the cache right after updating
		az.lbCache.Delete(*lb.Name)
		return nil
	}

	// Invalidate the cache because ETAG precondition mismatch.
	if rerr.HTTPStatusCode == http.StatusPreconditionFailed {
		klog.V(3).Infof("LoadBalancer cache for %s is cleanup because of http.StatusPreconditionFailed", pointer.StringDeref(lb.Name, ""))
		az.lbCache.Delete(*lb.Name)
	}

	retryErrorMessage := rerr.Error().Error()
	// Invalidate the cache because another new operation has canceled the current request.
	if strings.Contains(strings.ToLower(retryErrorMessage), operationCanceledErrorMessage) {
		klog.V(3).Infof("LoadBalancer cache for %s is cleanup because CreateOrUpdate is canceled by another operation", pointer.StringDeref(lb.Name, ""))
		az.lbCache.Delete(*lb.Name)
	}

	// The LB update may fail because the referenced PIP is not in the Succeeded provisioning state
	if strings.Contains(strings.ToLower(retryErrorMessage), strings.ToLower(referencedResourceNotProvisionedMessageCode)) {
		matches := pipErrorMessageRE.FindStringSubmatch(retryErrorMessage)
		if len(matches) != 3 {
			klog.Warningf("Failed to parse the retry error message %s", retryErrorMessage)
			return rerr.Error()
		}
		pipRG, pipName := matches[1], matches[2]
		klog.V(3).Infof("The public IP %s referenced by load balancer %s is not in Succeeded provisioning state, will try to update it", pipName, pointer.StringDeref(lb.Name, ""))
		pip, _, err := az.getPublicIPAddress(pipRG, pipName)
		if err != nil {
			klog.Warningf("Failed to get the public IP %s in resource group %s: %v", pipName, pipRG, err)
			return rerr.Error()
		}
		// Perform a dummy update to fix the provisioning state
		err = az.CreateOrUpdatePIP(service, pipRG, pip)
		if err != nil {
			klog.Warningf("Failed to update the public IP %s in resource group %s: %v", pipName, pipRG, err)
			return rerr.Error()
		}
		// Invalidate the LB cache, return the error, and the controller manager
		// would retry the LB update in the next reconcile loop
		az.lbCache.Delete(*lb.Name)
	}

	return rerr.Error()
}


ListLB

// ListLB invokes az.LoadBalancerClient.List with exponential backoff retry
func (az *Cloud) ListLB(service *v1.Service) ([]network.LoadBalancer, error) {
	ctx, cancel := getContextWithCancel()
	defer cancel()

	rgName := az.getLoadBalancerResourceGroup()
	allLBs, rerr := az.LoadBalancerClient.List(ctx, rgName)
	if rerr != nil {
		if rerr.IsNotFound() {
			return nil, nil
		}
		az.Event(service, v1.EventTypeWarning, "ListLoadBalancers", rerr.Error().Error())
		klog.Errorf("LoadBalancerClient.List(%v) failure with err=%v", rgName, rerr)
		return nil, rerr.Error()
	}
	klog.V(2).Infof("LoadBalancerClient.List(%v) success", rgName)
	return allLBs, nil
}


ListPIP

// ListPIP list the PIP resources in the given resource group
func (az *Cloud) ListPIP(service *v1.Service, pipResourceGroup string) ([]network.PublicIPAddress, error) {
	ctx, cancel := getContextWithCancel()
	defer cancel()

	allPIPs, rerr := az.PublicIPAddressesClient.List(ctx, pipResourceGroup)
	if rerr != nil {
		if rerr.IsNotFound() {
			return nil, nil
		}
		az.Event(service, v1.EventTypeWarning, "ListPublicIPs", rerr.Error().Error())
		klog.Errorf("PublicIPAddressesClient.List(%v) failure with err=%v", pipResourceGroup, rerr)
		return nil, rerr.Error()
	}

	klog.V(2).Infof("PublicIPAddressesClient.List(%v) success", pipResourceGroup)
	return allPIPs, nil
}


CreateOrUpdatePIP

// CreateOrUpdatePIP invokes az.PublicIPAddressesClient.CreateOrUpdate with exponential backoff retry
func (az *Cloud) CreateOrUpdatePIP(service *v1.Service, pipResourceGroup string, pip network.PublicIPAddress) error {
	ctx, cancel := getContextWithCancel()
	defer cancel()

	rerr := az.PublicIPAddressesClient.CreateOrUpdate(ctx, pipResourceGroup, pointer.StringDeref(pip.Name, ""), pip)
	klog.V(10).Infof("PublicIPAddressesClient.CreateOrUpdate(%s, %s): end", pipResourceGroup, pointer.StringDeref(pip.Name, ""))
	if rerr != nil {
		klog.Errorf("PublicIPAddressesClient.CreateOrUpdate(%s, %s) failed: %s", pipResourceGroup, pointer.StringDeref(pip.Name, ""), rerr.Error().Error())
		az.Event(service, v1.EventTypeWarning, "CreateOrUpdatePublicIPAddress", rerr.Error().Error())
		return rerr.Error()
	}

	return nil
}


CreateOrUpdateInterface

// CreateOrUpdateInterface invokes az.PublicIPAddressesClient.CreateOrUpdate with exponential backoff retry
func (az *Cloud) CreateOrUpdateInterface(service *v1.Service, nic network.Interface) error {
	ctx, cancel := getContextWithCancel()
	defer cancel()

	rerr := az.InterfacesClient.CreateOrUpdate(ctx, az.ResourceGroup, *nic.Name, nic)
	klog.V(10).Infof("InterfacesClient.CreateOrUpdate(%s): end", *nic.Name)
	if rerr != nil {
		klog.Errorf("InterfacesClient.CreateOrUpdate(%s) failed: %s", *nic.Name, rerr.Error().Error())
		az.Event(service, v1.EventTypeWarning, "CreateOrUpdateInterface", rerr.Error().Error())
		return rerr.Error()
	}

	return nil
}


DeletePublicIP

// DeletePublicIP invokes az.PublicIPAddressesClient.Delete with exponential backoff retry
func (az *Cloud) DeletePublicIP(service *v1.Service, pipResourceGroup string, pipName string) error {
	ctx, cancel := getContextWithCancel()
	defer cancel()

	rerr := az.PublicIPAddressesClient.Delete(ctx, pipResourceGroup, pipName)
	if rerr != nil {
		klog.Errorf("PublicIPAddressesClient.Delete(%s) failed: %s", pipName, rerr.Error().Error())
		az.Event(service, v1.EventTypeWarning, "DeletePublicIPAddress", rerr.Error().Error())

		if strings.Contains(rerr.Error().Error(), cannotDeletePublicIPErrorMessageCode) {
			klog.Warningf("DeletePublicIP for public IP %s failed with error %v, this is because other resources are referencing the public IP. The deletion of the service will continue.", pipName, rerr.Error())
			return nil
		}
		return rerr.Error()
	}

	return nil
}


DeleteLB

// DeleteLB invokes az.LoadBalancerClient.Delete with exponential backoff retry
func (az *Cloud) DeleteLB(service *v1.Service, lbName string) error {
	ctx, cancel := getContextWithCancel()
	defer cancel()

	rgName := az.getLoadBalancerResourceGroup()
	rerr := az.LoadBalancerClient.Delete(ctx, rgName, lbName)
	if rerr == nil {
		// Invalidate the cache right after updating
		az.lbCache.Delete(lbName)
		return nil
	}

	klog.Errorf("LoadBalancerClient.Delete(%s) failed: %s", lbName, rerr.Error().Error())
	az.Event(service, v1.EventTypeWarning, "DeleteLoadBalancer", rerr.Error().Error())
	return rerr.Error()
}


CreateOrUpdateRouteTable

// CreateOrUpdateRouteTable invokes az.RouteTablesClient.CreateOrUpdate with exponential backoff retry
func (az *Cloud) CreateOrUpdateRouteTable(routeTable network.RouteTable) error {
	ctx, cancel := getContextWithCancel()
	defer cancel()

	rerr := az.RouteTablesClient.CreateOrUpdate(ctx, az.RouteTableResourceGroup, az.RouteTableName, routeTable, pointer.StringDeref(routeTable.Etag, ""))
	if rerr == nil {
		// Invalidate the cache right after updating
		az.rtCache.Delete(*routeTable.Name)
		return nil
	}

	// Invalidate the cache because etag mismatch.
	if rerr.HTTPStatusCode == http.StatusPreconditionFailed {
		klog.V(3).Infof("Route table cache for %s is cleanup because of http.StatusPreconditionFailed", *routeTable.Name)
		az.rtCache.Delete(*routeTable.Name)
	}
	// Invalidate the cache because another new operation has canceled the current request.
	if strings.Contains(strings.ToLower(rerr.Error().Error()), operationCanceledErrorMessage) {
		klog.V(3).Infof("Route table cache for %s is cleanup because CreateOrUpdateRouteTable is canceled by another operation", *routeTable.Name)
		az.rtCache.Delete(*routeTable.Name)
	}
	klog.Errorf("RouteTablesClient.CreateOrUpdate(%s) failed: %v", az.RouteTableName, rerr.Error())
	return rerr.Error()
}


CreateOrUpdateRoute

// CreateOrUpdateRoute invokes az.RoutesClient.CreateOrUpdate with exponential backoff retry
func (az *Cloud) CreateOrUpdateRoute(route network.Route) error {
	ctx, cancel := getContextWithCancel()
	defer cancel()

	rerr := az.RoutesClient.CreateOrUpdate(ctx, az.RouteTableResourceGroup, az.RouteTableName, *route.Name, route, pointer.StringDeref(route.Etag, ""))
	klog.V(10).Infof("RoutesClient.CreateOrUpdate(%s): end", *route.Name)
	if rerr == nil {
		az.rtCache.Delete(az.RouteTableName)
		return nil
	}

	if rerr.HTTPStatusCode == http.StatusPreconditionFailed {
		klog.V(3).Infof("Route cache for %s is cleanup because of http.StatusPreconditionFailed", *route.Name)
		az.rtCache.Delete(az.RouteTableName)
	}
	// Invalidate the cache because another new operation has canceled the current request.
	if strings.Contains(strings.ToLower(rerr.Error().Error()), operationCanceledErrorMessage) {
		klog.V(3).Infof("Route cache for %s is cleanup because CreateOrUpdateRouteTable is canceled by another operation", *route.Name)
		az.rtCache.Delete(az.RouteTableName)
	}
	return rerr.Error()
}


DeleteRouteWithName

// DeleteRouteWithName invokes az.RoutesClient.CreateOrUpdate with exponential backoff retry
func (az *Cloud) DeleteRouteWithName(routeName string) error {
	ctx, cancel := getContextWithCancel()
	defer cancel()

	rerr := az.RoutesClient.Delete(ctx, az.RouteTableResourceGroup, az.RouteTableName, routeName)
	klog.V(10).Infof("RoutesClient.Delete(%s,%s): end", az.RouteTableName, routeName)
	if rerr == nil {
		return nil
	}

	klog.Errorf("RoutesClient.Delete(%s, %s) failed: %v", az.RouteTableName, routeName, rerr.Error())
	return rerr.Error()
}


CreateOrUpdateVMSS

// CreateOrUpdateVMSS invokes az.VirtualMachineScaleSetsClient.Update().
func (az *Cloud) CreateOrUpdateVMSS(resourceGroupName string, VMScaleSetName string, parameters compute.VirtualMachineScaleSet) *retry.Error {
	ctx, cancel := getContextWithCancel()
	defer cancel()

	// When vmss is being deleted, CreateOrUpdate API would report "the vmss is being deleted" error.
	// Since it is being deleted, we shouldn't send more CreateOrUpdate requests for it.
	klog.V(3).Infof("CreateOrUpdateVMSS: verify the status of the vmss being created or updated")
	vmss, rerr := az.VirtualMachineScaleSetsClient.Get(ctx, resourceGroupName, VMScaleSetName)
	if rerr != nil {
		klog.Errorf("CreateOrUpdateVMSS: error getting vmss(%s): %v", VMScaleSetName, rerr)
		return rerr
	}
	if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, virtualMachineScaleSetsDeallocating) {
		klog.V(3).Infof("CreateOrUpdateVMSS: found vmss %s being deleted, skipping", VMScaleSetName)
		return nil
	}

	rerr = az.VirtualMachineScaleSetsClient.CreateOrUpdate(ctx, resourceGroupName, VMScaleSetName, parameters)
	klog.V(10).Infof("UpdateVmssVMWithRetry: VirtualMachineScaleSetsClient.CreateOrUpdate(%s): end", VMScaleSetName)
	if rerr != nil {
		klog.Errorf("CreateOrUpdateVMSS: error CreateOrUpdate vmss(%s): %v", VMScaleSetName, rerr)
		return rerr
	}

	return nil
}


initStorageAccounts

func (c *BlobDiskController) initStorageAccounts() {
	accountsLock.Lock()
	defer accountsLock.Unlock()

	if c.accounts == nil {
		// get accounts
		accounts, err := c.getAllStorageAccounts()
		if err != nil {
			klog.Errorf("azureDisk - getAllStorageAccounts error: %v", err)
			c.accounts = make(map[string]*storageAccountState)
		}
		c.accounts = accounts
	}
}


CreateVolume

// CreateVolume creates a VHD blob in a storage account that has storageType and location using the given storage account.
// If no storage account is given, search all the storage accounts associated with the resource group and pick one that
// fits storage type and location.
func (c *BlobDiskController) CreateVolume(blobName, accountName, accountType, location string, requestGB int) (string, string, int, error) {
	accountOptions := &AccountOptions{
		Name:                   accountName,
		Type:                   accountType,
		Kind:                   string(defaultStorageAccountKind),
		ResourceGroup:          c.common.resourceGroup,
		Location:               location,
		EnableHTTPSTrafficOnly: true,
	}
	account, key, err := c.common.cloud.EnsureStorageAccount(accountOptions, dedicatedDiskAccountNamePrefix)
	if err != nil {
		return "", "", 0, fmt.Errorf("could not get storage key for storage account %s: %v", accountName, err)
	}

	client, err := azstorage.NewBasicClientOnSovereignCloud(account, key, c.common.cloud.Environment)
	if err != nil {
		return "", "", 0, err
	}
	blobClient := client.GetBlobService()

	// create a page blob in this account's vhd container
	diskName, diskURI, err := c.createVHDBlobDisk(blobClient, account, blobName, vhdContainerName, int64(requestGB))
	if err != nil {
		return "", "", 0, err
	}

	klog.V(4).Infof("azureDisk - created vhd blob uri: %s", diskURI)
	return diskName, diskURI, requestGB, err
}


DeleteVolume

// DeleteVolume deletes a VHD blob
func (c *BlobDiskController) DeleteVolume(diskURI string) error {
	klog.V(4).Infof("azureDisk - begin to delete volume %s", diskURI)
	accountName, blob, err := c.common.cloud.getBlobNameAndAccountFromURI(diskURI)
	if err != nil {
		return fmt.Errorf("failed to parse vhd URI %v", err)
	}
	key, err := c.common.cloud.GetStorageAccesskey(accountName, c.common.resourceGroup)
	if err != nil {
		return fmt.Errorf("no key for storage account %s, err %v", accountName, err)
	}
	err = c.common.cloud.deleteVhdBlob(accountName, key, blob)
	if err != nil {
		klog.Warningf("azureDisk - failed to delete blob %s err: %v", diskURI, err)
		detail := err.Error()
		if strings.Contains(detail, errLeaseIDMissing) {
			// disk is still being used
			// see https://msdn.microsoft.com/en-us/library/microsoft.windowsazure.storage.blob.protocol.bloberrorcodestrings.leaseidmissing.aspx
			return volerr.NewDeletedVolumeInUseError(fmt.Sprintf("disk %q is still in use while being deleted", diskURI))
		}
		return fmt.Errorf("failed to delete vhd %v, account %s, blob %s, err: %v", diskURI, accountName, blob, err)
	}
	klog.V(4).Infof("azureDisk - blob %s deleted", diskURI)
	return nil

}


getBlobNameAndAccountFromURI

// get diskURI https://foo.blob.core.windows.net/vhds/bar.vhd and return foo (account) and bar.vhd (blob name)
func (c *BlobDiskController) getBlobNameAndAccountFromURI(diskURI string) (string, string, error) {
	scheme := "http"
	if useHTTPSForBlobBasedDisk {
		scheme = "https"
	}
	host := fmt.Sprintf("%s://(.*).%s.%s", scheme, blobServiceName, c.common.storageEndpointSuffix)
	reStr := fmt.Sprintf("%s/%s/(.*)", host, vhdContainerName)
	re := regexp.MustCompile(reStr)
	res := re.FindSubmatch([]byte(diskURI))
	if len(res) < 3 {
		return "", "", fmt.Errorf("invalid vhd URI for regex %s: %s", reStr, diskURI)
	}
	return string(res[1]), string(res[2]), nil
}