main

func main() {
	if len(os.Args) != 3 {
		log.Fatalln("usage: wincat <host> <port>")
	}
	host := os.Args[1]
	port := os.Args[2]

	addr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%s", host, port))
	if err != nil {
		log.Fatalf("Failed to resolve TCP addr %v %v", host, port)
	}

	conn, err := net.DialTCP("tcp", nil, addr)
	if err != nil {
		log.Fatalf("Failed to connect to %s:%s because %s", host, port, err)
	}
	defer conn.Close()

	var wg sync.WaitGroup
	wg.Add(2)

	go func() {
		defer func() {
			os.Stdout.Close()
			os.Stdin.Close()
			conn.CloseRead()
			wg.Done()
		}()

		_, err := io.Copy(os.Stdout, conn)
		if err != nil {
			log.Printf("error while copying stream to stdout: %v", err)
		}
	}()

	go func() {
		defer func() {
			conn.CloseWrite()
			wg.Done()
		}()

		_, err := io.Copy(conn, os.Stdin)
		if err != nil {
			log.Printf("error while copying stream from stdin: %v", err)
		}
	}()

	wg.Wait()
}


func main() {

	if len(os.Args) < 2 {
		fmt.Fprintf(os.Stderr, "Command failed: must provide a command to run.\n")
		return
	}
	path, _ := filepath.Split(os.Args[0])
	rootfsPath := filepath.Join(path, rootfs)
	if _, err := os.Stat(rootfsPath); os.IsNotExist(err) {
		rootfsPath = defaultRootfs
	}
	command := os.Args[1]
	switch command {
	case mountCmd:
		mountErr := mountInChroot(rootfsPath, os.Args[2:])
		if mountErr != nil {
			fmt.Fprintf(os.Stderr, "Mount failed: %v", mountErr)
			os.Exit(1)
		}
	default:
		fmt.Fprintf(os.Stderr, "Unknown command, must be %s", mountCmd)
		os.Exit(1)

	}
}


mountInChroot

// MountInChroot is to run mount within chroot with the passing root directory
func mountInChroot(rootfsPath string, args []string) error {
	if _, err := os.Stat(rootfsPath); os.IsNotExist(err) {
		return fmt.Errorf("path <%s> does not exist", rootfsPath)
	}
	args = append([]string{rootfsPath, mountCmd}, args...)
	output, err := exec.Command(chrootCmd, args...).CombinedOutput()
	if err == nil {
		return nil
	}

	if !strings.EqualFold(string(output), nfsRPCBindErrMsg) {
		// Mount failed but not because of RPC bind error
		return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %v\nOutput: %s", err, chrootCmd, args, string(output))
	}

	// Mount failed because it is NFS V3 and we need to run rpcBind
	output, err = exec.Command(chrootCmd, rootfsPath, rpcBindCmd, "-w").CombinedOutput()
	if err != nil {
		return fmt.Errorf("mount issued for NFS V3 but unable to run rpcbind:\n Output: %s\n Error: %v", string(output), err)
	}

	// Rpcbind is running, try mounting again
	output, err = exec.Command(chrootCmd, args...).CombinedOutput()

	if err != nil {
		return fmt.Errorf("mount failed for NFS V3 even after running rpcBind %s, %v", string(output), err)
	}

	return nil
}


registerFlags

func registerFlags(fs *pflag.FlagSet) {
	fs.StringVar(&listenAddress, "listen-address", "localhost:9101", "Address to listen on for serving prometheus metrics")
	fs.StringVar(&metricsPath, "metrics-path", "/metrics", "Path under which prometheus metrics are to be served")
	fs.StringVar(&etcdVersionScrapeURI, "etcd-version-scrape-uri", "http://localhost:2379/version", "URI to scrape etcd version info")
	fs.StringVar(&etcdMetricsScrapeURI, "etcd-metrics-scrape-uri", "http://localhost:2379/metrics", "URI to scrape etcd metrics")
	fs.DurationVar(&scrapeTimeout, "scrape-timeout", 15*time.Second, "Timeout for trying to get stats from etcd")
}


Gather

func (m *monitorGatherer) Gather() ([]*dto.MetricFamily, error) {
	etcdMetrics, err := scrapeMetrics()
	if err != nil {
		return nil, err
	}
	exported, err := m.rewriteExportedMetrics(etcdMetrics)
	if err != nil {
		return nil, err
	}
	custom, err := customMetricRegistry.Gather()
	if err != nil {
		return nil, err
	}
	result := make([]*dto.MetricFamily, 0, len(exported)+len(custom))
	result = append(result, exported...)
	result = append(result, custom...)
	return result, nil
}


rewriteExportedMetrics

func (m *monitorGatherer) rewriteExportedMetrics(metrics map[string]*dto.MetricFamily) ([]*dto.MetricFamily, error) {
	results := make([]*dto.MetricFamily, 0, len(metrics))
	for n, mf := range metrics {
		if e, ok := m.exported[n]; ok {
			// Apply rewrite rules for metrics that have them.
			if e.rewriters == nil {
				results = append(results, mf)
			} else {
				for _, rewriter := range e.rewriters {
					new, err := rewriter(mf)
					if err != nil {
						return nil, err
					}
					results = append(results, new)
				}
			}
		} else {
			// Proxy all metrics without any rewrite rules directly.
			results = append(results, mf)
		}
	}
	return results, nil
}


getVersion

// Function for fetching etcd version info and feeding it to the prometheus metric.
func getVersion(lastSeenBinaryVersion *string) error {
	// Create the get request for the etcd version endpoint.
	req, err := http.NewRequest("GET", etcdVersionScrapeURI, nil)
	if err != nil {
		return fmt.Errorf("failed to create GET request for etcd version: %v", err)
	}

	// Send the get request and receive a response.
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return fmt.Errorf("failed to receive GET response for etcd version: %v", err)
	}
	defer resp.Body.Close()

	// Obtain EtcdVersion from the JSON response.
	var version EtcdVersion
	if err := json.NewDecoder(resp.Body).Decode(&version); err != nil {
		return fmt.Errorf("failed to decode etcd version JSON: %v", err)
	}

	// Return without updating the version if it stayed the same since last time.
	if *lastSeenBinaryVersion == version.BinaryVersion {
		return nil
	}

	// Delete the metric for the previous version.
	if *lastSeenBinaryVersion != "" {
		deleted := etcdVersion.Delete(metrics.Labels{"binary_version": *lastSeenBinaryVersion})
		if !deleted {
			return errors.New("failed to delete previous version's metric")
		}
	}

	// Record the new version in a metric.
	etcdVersion.With(metrics.Labels{
		"binary_version": version.BinaryVersion,
	}).Set(0)
	*lastSeenBinaryVersion = version.BinaryVersion
	return nil
}


getVersionPeriodically

// Periodically fetches etcd version info.
func getVersionPeriodically(stopCh <-chan struct{}) {
	lastSeenBinaryVersion := ""
	for {
		if err := getVersion(&lastSeenBinaryVersion); err != nil {
			klog.Errorf("Failed to fetch etcd version: %v", err)
		}
		select {
		case <-stopCh:
			return
		case <-time.After(scrapeTimeout):
		}
	}
}


scrapeMetrics

// scrapeMetrics scrapes the prometheus metrics from the etcd metrics URI.
func scrapeMetrics() (map[string]*dto.MetricFamily, error) {
	req, err := http.NewRequest("GET", etcdMetricsScrapeURI, nil)
	if err != nil {
		return nil, fmt.Errorf("failed to create GET request for etcd metrics: %v", err)
	}

	// Send the get request and receive a response.
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("failed to receive GET response for etcd metrics: %v", err)
	}
	defer resp.Body.Close()

	return testutil.TextToMetricFamilies(resp.Body)
}


renameMetric

func renameMetric(mf *dto.MetricFamily, name string) {
	mf.Name = &name
}


renameLabels

func renameLabels(mf *dto.MetricFamily, nameMapping map[string]string) {
	for _, m := range mf.Metric {
		for _, lbl := range m.Label {
			if alias, ok := nameMapping[*lbl.Name]; ok {
				lbl.Name = &alias
			}
		}
	}
}


filterMetricsByLabels

func filterMetricsByLabels(mf *dto.MetricFamily, labelValues map[string]string) {
	buf := mf.Metric[:0]
	for _, m := range mf.Metric {
		shouldRemove := false
		for _, lbl := range m.Label {
			if val, ok := labelValues[*lbl.Name]; ok && val != *lbl.Value {
				shouldRemove = true
				break
			}
		}
		if !shouldRemove {
			buf = append(buf, m)
		}
	}
	mf.Metric = buf
}


groupCounterMetricsByLabels

func groupCounterMetricsByLabels(mf *dto.MetricFamily, names map[string]bool) {
	buf := mf.Metric[:0]
	deleteLabels(mf, names)
	byLabels := map[string]*dto.Metric{}
	for _, m := range mf.Metric {
		if metric, ok := byLabels[labelsKey(m.Label)]; ok {
			metric.Counter.Value = proto.Float64(*metric.Counter.Value + *m.Counter.Value)
		} else {
			byLabels[labelsKey(m.Label)] = m
			buf = append(buf, m)
		}
	}
	mf.Metric = buf
}


labelsKey

func labelsKey(lbls []*dto.LabelPair) string {
	var buf bytes.Buffer
	for i, lbl := range lbls {
		buf.WriteString(lbl.String())
		if i < len(lbls)-1 {
			buf.WriteString(",")
		}
	}
	return buf.String()
}


deleteLabels

func deleteLabels(mf *dto.MetricFamily, names map[string]bool) {
	for _, m := range mf.Metric {
		buf := m.Label[:0]
		for _, lbl := range m.Label {
			shouldRemove := names[*lbl.Name]
			if !shouldRemove {
				buf = append(buf, lbl)
			}
		}
		m.Label = buf
	}
}


identity

func identity(mf *dto.MetricFamily) (*dto.MetricFamily, error) {
	return mf, nil
}


deepCopyMetricFamily

func deepCopyMetricFamily(mf *dto.MetricFamily) *dto.MetricFamily {
	r := &dto.MetricFamily{}
	r.Name = mf.Name
	r.Help = mf.Help
	r.Type = mf.Type
	r.Metric = make([]*dto.Metric, len(mf.Metric))
	for i, m := range mf.Metric {
		r.Metric[i] = deepCopyMetric(m)
	}
	return r
}


deepCopyMetric

func deepCopyMetric(m *dto.Metric) *dto.Metric {
	r := &dto.Metric{}
	r.Label = make([]*dto.LabelPair, len(m.Label))
	for i, lp := range m.Label {
		r.Label[i] = deepCopyLabelPair(lp)
	}
	r.Gauge = m.Gauge
	r.Counter = m.Counter
	r.Summary = m.Summary
	r.Untyped = m.Untyped
	r.Histogram = m.Histogram
	r.TimestampMs = m.TimestampMs
	return r
}


deepCopyLabelPair

func deepCopyLabelPair(lp *dto.LabelPair) *dto.LabelPair {
	r := &dto.LabelPair{}
	r.Name = lp.Name
	r.Value = lp.Value
	return r
}


main

func main() {
	// Register the commandline flags passed to the tool.
	registerFlags(pflag.CommandLine)
	pflag.CommandLine.AddGoFlagSet(goflag.CommandLine)
	pflag.Parse()

	// Register the metrics we defined above with prometheus.
	customMetricRegistry.MustRegister(etcdVersion)

	// Spawn threads for periodically scraping etcd version metrics.
	stopCh := make(chan struct{})
	defer close(stopCh)
	go getVersionPeriodically(stopCh)

	// Serve our metrics on listenAddress/metricsPath.
	klog.Infof("Listening on: %v", listenAddress)
	http.Handle(metricsPath, metrics.HandlerFor(gatherer, metrics.HandlerOpts{}))
	klog.Errorf("Stopped listening/serving metrics: %v", http.ListenAndServe(listenAddress, nil))
}


copyFile

func copyFile(source, dest string) error {
	sf, err := os.Open(source)
	if err != nil {
		return fmt.Errorf("unable to open source file [%s]: %q", source, err)
	}
	defer sf.Close()
	fi, err := sf.Stat()
	if err != nil {
		return fmt.Errorf("unable to stat source file [%s]: %q", source, err)
	}

	dir := filepath.Dir(dest)
	if err := os.MkdirAll(dir, 0755); err != nil {
		return fmt.Errorf("unable to create directory [%s]: %q", dir, err)
	}
	df, err := os.Create(dest)
	if err != nil {
		return fmt.Errorf("unable to create destination file [%s]: %q", dest, err)
	}
	defer df.Close()

	_, err = io.Copy(df, sf)
	if err != nil {
		return fmt.Errorf("unable to copy [%s] to [%s]: %q", source, dest, err)
	}

	if err := os.Chmod(dest, fi.Mode()); err != nil {
		return fmt.Errorf("unable to close destination file: %q", err)
	}
	return nil
}


OpenOrCreateDataDirectory

// OpenOrCreateDataDirectory opens a data directory, creating the directory
// if it doesn't not already exist.
func OpenOrCreateDataDirectory(path string) (*DataDirectory, error) {
	exists, err := exists(path)
	if err != nil {
		return nil, err
	}
	if !exists {
		klog.Infof("data directory '%s' does not exist, creating it", path)
		err := os.MkdirAll(path, 0777)
		if err != nil {
			return nil, fmt.Errorf("failed to create data directory %s: %v", path, err)
		}
	}
	versionFile := &VersionFile{
		path: filepath.Join(path, versionFilename),
	}
	return &DataDirectory{path, versionFile}, nil
}


Initialize

// Initialize set the version.txt to the target version if the data
// directory is empty. If the data directory is non-empty, no
// version.txt file will be written since the actual version of etcd
// used to create the data is unknown.
func (d *DataDirectory) Initialize(target *EtcdVersionPair) error {
	isEmpty, err := d.IsEmpty()
	if err != nil {
		return err
	}
	if isEmpty {
		klog.Infof("data directory '%s' is empty, writing target version '%s' to version.txt", d.path, target)
		err = d.versionFile.Write(target)
		if err != nil {
			return fmt.Errorf("failed to write version.txt to '%s': %v", d.path, err)
		}
		return nil
	}
	return nil
}


Backup

// Backup creates a backup copy of data directory.
func (d *DataDirectory) Backup() error {
	backupDir := fmt.Sprintf("%s.bak", d.path)
	err := os.RemoveAll(backupDir)
	if err != nil {
		return err
	}
	err = os.MkdirAll(backupDir, 0777)
	if err != nil {
		return err
	}
	err = copyDirectory(d.path, backupDir)
	if err != nil {
		return err
	}

	return nil
}


IsEmpty

// IsEmpty returns true if the data directory is entirely empty.
func (d *DataDirectory) IsEmpty() (bool, error) {
	dir, err := os.Open(d.path)
	if err != nil {
		return false, fmt.Errorf("failed to open data directory %s: %v", d.path, err)
	}
	defer dir.Close()
	_, err = dir.Readdirnames(1)
	if err == io.EOF {
		return true, nil
	}
	return false, err
}


String

// String returns the data directory path.
func (d *DataDirectory) String() string {
	return d.path
}


nextPath

func (v *VersionFile) nextPath() string {
	return fmt.Sprintf("%s-next", v.path)
}


Exists

// Exists returns true if a version.txt file exists on the file system.
func (v *VersionFile) Exists() (bool, error) {
	return exists(v.path)
}


Read

// Read parses the version.txt file and returns it's contents.
func (v *VersionFile) Read() (*EtcdVersionPair, error) {
	data, err := os.ReadFile(v.path)
	if err != nil {
		return nil, fmt.Errorf("failed to read version file %s: %v", v.path, err)
	}
	txt := strings.TrimSpace(string(data))
	vp, err := ParseEtcdVersionPair(txt)
	if err != nil {
		return nil, fmt.Errorf("failed to parse etcd '<version>/<storage-version>' string from version.txt file contents '%s': %v", txt, err)
	}
	return vp, nil
}


equals

// equals returns true iff VersionFile exists and contains given EtcdVersionPair.
func (v *VersionFile) equals(vp *EtcdVersionPair) (bool, error) {
	exists, err := v.Exists()
	if err != nil {
		return false, err
	}
	if !exists {
		return false, nil
	}
	cvp, err := v.Read()
	if err != nil {
		return false, err
	}
	return vp.Equals(cvp), nil
}


Write

// Write creates or overwrites the contents of the version.txt file with the given EtcdVersionPair.
func (v *VersionFile) Write(vp *EtcdVersionPair) error {
	// We do write only if file content differs from given EtcdVersionPair.
	isUpToDate, err := v.equals(vp)
	if err != nil {
		return fmt.Errorf("failed to to check if version file %s should be changed: %v", v.path, err)
	}
	if isUpToDate {
		return nil
	}
	// We do write + rename instead of just write to protect from version.txt
	// corruption under full disk condition.
	// See https://github.com/kubernetes/kubernetes/issues/98989.
	err = os.WriteFile(v.nextPath(), []byte(vp.String()), 0666)
	if err != nil {
		return fmt.Errorf("failed to write new version file %s: %v", v.nextPath(), err)
	}
	return os.Rename(v.nextPath(), v.path)
}


exists

func exists(path string) (bool, error) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return false, nil
	} else if err != nil {
		return false, err
	}
	return true, nil
}


main

func main() {
	registerFlags(migrateCmd.Flags(), &opts)
	err := migrateCmd.Execute()
	if err != nil {
		klog.Errorf("Failed to execute migratecmd: %s", err)
	}
}


runMigrate

// runMigrate starts the migration.
func runMigrate() {
	if err := opts.validateAndDefault(); err != nil {
		klog.Fatalf("%v", err)
	}
	copyBinaries()

	target := &EtcdVersionPair{
		version:        MustParseEtcdVersion(opts.targetVersion),
		storageVersion: MustParseEtcdStorageVersion(opts.targetStorage),
	}

	migrate(
		opts.name, opts.port, opts.peerListenUrls, opts.peerAdvertiseUrls, opts.clientListenUrls,
		opts.binDir, opts.dataDir, opts.etcdDataPrefix, opts.ttlKeysDirectory, opts.initialCluster,
		target, opts.supportedVersions, opts.etcdServerArgs)
}


copyBinaries

func copyBinaries() {
	if val, err := lookupEnv("DO_NOT_MOVE_BINARIES"); err != nil || val != "true" {
		etcdVersioned := fmt.Sprintf("etcd-%s", opts.targetVersion)
		etcdctlVersioned := fmt.Sprintf("etcdctl-%s", opts.targetVersion)
		if err := copyFile(filepath.Join(opts.binDir, etcdVersioned), filepath.Join(opts.binDir, "etcd")); err != nil {
			klog.Fatalf("Failed to copy %s: %v", etcdVersioned, err)
		}
		if err := copyFile(filepath.Join(opts.binDir, etcdctlVersioned), filepath.Join(opts.binDir, "etcdctl")); err != nil {
			klog.Fatalf("Failed to copy %s: %v", etcdctlVersioned, err)
		}
	}
}


migrate

// migrate opens or initializes the etcd data directory, configures the migrator, and starts the migration.
func migrate(name string, port uint64, peerListenUrls string, peerAdvertiseUrls string, clientListenUrls string,
	binPath string, dataDirPath string, etcdDataPrefix string, ttlKeysDirectory string,
	initialCluster string, target *EtcdVersionPair, bundledVersions SupportedVersions, etcdServerArgs string) {

	dataDir, err := OpenOrCreateDataDirectory(dataDirPath)
	if err != nil {
		klog.Fatalf("Error opening or creating data directory %s: %v", dataDirPath, err)
	}

	cfg := &EtcdMigrateCfg{
		binPath:           binPath,
		name:              name,
		port:              port,
		peerListenUrls:    peerListenUrls,
		peerAdvertiseUrls: peerAdvertiseUrls,
		clientListenUrls:  clientListenUrls,
		etcdDataPrefix:    etcdDataPrefix,
		ttlKeysDirectory:  ttlKeysDirectory,
		initialCluster:    initialCluster,
		supportedVersions: bundledVersions,
		dataDirectory:     dataDirPath,
		etcdServerArgs:    etcdServerArgs,
	}
	client, err := NewEtcdMigrateClient(cfg)
	if err != nil {
		klog.Fatalf("Migration failed: %v", err)
	}
	defer client.Close()

	migrator := &Migrator{cfg, dataDir, client}

	err = migrator.MigrateIfNeeded(target)
	if err != nil {
		klog.Fatalf("Migration failed: %v", err)
	}
}


NewEtcdMigrateClient

// NewEtcdMigrateClient creates a new EtcdMigrateClient from the given EtcdMigrateCfg.
func NewEtcdMigrateClient(cfg *EtcdMigrateCfg) (EtcdMigrateClient, error) {
	return &CombinedEtcdClient{cfg}, nil
}


Close

// Close closes the client and releases any resources it holds.
func (e *CombinedEtcdClient) Close() error {
	return nil
}


SetEtcdVersionKeyValue

// SetEtcdVersionKeyValue writes the given version to the etcd 'etcd_version' key.
// If no error is returned, the write was successful, indicating the etcd server is available
// and able to perform consensus writes.
func (e *CombinedEtcdClient) SetEtcdVersionKeyValue(version *EtcdVersion) error {
	return e.Put(version, "etcd_version", version.String())
}


Put

// Put write a single key value pair to etcd.
func (e *CombinedEtcdClient) Put(version *EtcdVersion, key, value string) error {
	v3client, err := e.clientV3()
	if err != nil {
		return err
	}
	defer v3client.Close()
	_, err = v3client.KV.Put(context.Background(), key, value)
	return err
}


Get

// Get reads a single value for a given key.
func (e *CombinedEtcdClient) Get(version *EtcdVersion, key string) (string, error) {
	v3client, err := e.clientV3()
	if err != nil {
		return "", err
	}
	defer v3client.Close()
	resp, err := v3client.KV.Get(context.Background(), key)
	if err != nil {
		return "", err
	}
	kvs := resp.Kvs
	if len(kvs) != 1 {
		return "", fmt.Errorf("expected exactly one value for key %s but got %d", key, len(kvs))
	}

	return string(kvs[0].Value), nil
}


func (e *CombinedEtcdClient) clientV3() (*clientv3.Client, error) {
	return clientv3.New(clientv3.Config{
		Endpoints:   []string{e.endpoint()},
		DialTimeout: 20 * time.Second,
		DialOptions: []grpc.DialOption{
			grpc.WithBlock(), // block until the underlying connection is up
		},
	})
}


Backup

// Backup creates a backup of an etcd2 data directory at the given backupDir.
func (e *CombinedEtcdClient) Backup(version *EtcdVersion, backupDir string) error {
	// We cannot use etcd/client (v2) to make this call. It is implemented in the etcdctl client code.
	if version.Major != 2 {
		return fmt.Errorf("etcd 2.x required but got version '%s'", version)
	}
	return e.runEtcdctlCommand(version,
		"--debug",
		"backup",
		"--data-dir", e.cfg.dataDirectory,
		"--backup-dir", backupDir,
	)
}


Snapshot

// Snapshot captures a snapshot from a running etcd3 server and saves it to the given snapshotFile.
// We cannot use etcd/clientv3 to make this call. It is implemented in the etcdctl client code.
func (e *CombinedEtcdClient) Snapshot(version *EtcdVersion, snapshotFile string) error {
	if version.Major != 3 {
		return fmt.Errorf("etcd 3.x required but got version '%s'", version)
	}
	return e.runEtcdctlCommand(version,
		"--endpoints", e.endpoint(),
		"snapshot", "save", snapshotFile,
	)
}


Restore

// Restore restores a given snapshotFile into the data directory specified this clients config.
func (e *CombinedEtcdClient) Restore(version *EtcdVersion, snapshotFile string) error {
	// We cannot use etcd/clientv3 to make this call. It is implemented in the etcdctl client code.
	if version.Major != 3 {
		return fmt.Errorf("etcd 3.x required but got version '%s'", version)
	}
	return e.runEtcdctlCommand(version,
		"snapshot", "restore", snapshotFile,
		"--data-dir", e.cfg.dataDirectory,
		"--name", e.cfg.name,
		"--initial-advertise-peer-urls", e.cfg.peerAdvertiseUrls,
		"--initial-cluster", e.cfg.initialCluster,
	)
}


Migrate

// Migrate upgrades a 'etcd2' storage version data directory to a 'etcd3' storage version
// data directory.
func (e *CombinedEtcdClient) Migrate(version *EtcdVersion) error {
	// We cannot use etcd/clientv3 to make this call as it is implemented in etcd/etcdctl.
	if version.Major != 3 {
		return fmt.Errorf("etcd 3.x required but got version '%s'", version)
	}
	return e.runEtcdctlCommand(version,
		"migrate",
		"--data-dir", e.cfg.dataDirectory,
	)
}


runEtcdctlCommand

func (e *CombinedEtcdClient) runEtcdctlCommand(version *EtcdVersion, args ...string) error {
	etcdctlCmd := exec.Command(filepath.Join(e.cfg.binPath, fmt.Sprintf("etcdctl-%s", version)), args...)
	etcdctlCmd.Env = []string{fmt.Sprintf("ETCDCTL_API=%d", version.Major)}
	etcdctlCmd.Stdout = os.Stdout
	etcdctlCmd.Stderr = os.Stderr
	return etcdctlCmd.Run()
}


AttachLease

// AttachLease attaches leases of the given leaseDuration to all the  etcd objects under
// ttlKeysDirectory specified in this client's config.
func (e *CombinedEtcdClient) AttachLease(leaseDuration time.Duration) error {
	ttlKeysPrefix := e.cfg.ttlKeysDirectory
	// Make sure that ttlKeysPrefix is ended with "/" so that we only get children "directories".
	if !strings.HasSuffix(ttlKeysPrefix, "/") {
		ttlKeysPrefix += "/"
	}
	ctx := context.Background()

	v3client, err := e.clientV3()
	if err != nil {
		return err
	}
	defer v3client.Close()
	objectsResp, err := v3client.KV.Get(ctx, ttlKeysPrefix, clientv3.WithPrefix())
	if err != nil {
		return fmt.Errorf("error while getting objects to attach to the lease")
	}

	lease, err := v3client.Lease.Grant(ctx, int64(leaseDuration/time.Second))
	if err != nil {
		return fmt.Errorf("error while creating lease: %v", err)
	}
	klog.Infof("Lease with TTL: %v created", lease.TTL)

	klog.Infof("Attaching lease to %d entries", len(objectsResp.Kvs))
	for _, kv := range objectsResp.Kvs {
		putResp, err := v3client.KV.Put(ctx, string(kv.Key), string(kv.Value), clientv3.WithLease(lease.ID), clientv3.WithPrevKV())
		if err != nil {
			klog.Errorf("Error while attaching lease to: %s", string(kv.Key))
		}
		if !bytes.Equal(putResp.PrevKv.Value, kv.Value) {
			return fmt.Errorf("concurrent access to key detected when setting lease on %s, expected previous value of %s but got %s",
				kv.Key, kv.Value, putResp.PrevKv.Value)
		}
	}
	return nil
}


endpoint

func (e *CombinedEtcdClient) endpoint() string {
	return fmt.Sprintf("http://127.0.0.1:%d", e.cfg.port)
}


NewEtcdMigrateServer

// NewEtcdMigrateServer creates a EtcdMigrateServer for starting and stopping a etcd server at the given version.
func NewEtcdMigrateServer(cfg *EtcdMigrateCfg, client EtcdMigrateClient) *EtcdMigrateServer {
	return &EtcdMigrateServer{cfg: cfg, client: client}
}


Start

// Start starts an etcd server as a separate process, waits until it has started, and returns a exec.Cmd.
// TODO: Add support for listening to client via TLS.
func (r *EtcdMigrateServer) Start(version *EtcdVersion) error {
	etcdCmd := exec.Command(
		fmt.Sprintf("%s/etcd-%s", r.cfg.binPath, version),
		"--name", r.cfg.name,
		"--initial-cluster", r.cfg.initialCluster,
		"--debug",
		"--data-dir", r.cfg.dataDirectory,
		"--listen-client-urls", r.cfg.clientListenUrls,
		"--advertise-client-urls", fmt.Sprintf("http://127.0.0.1:%d", r.cfg.port),
		"--listen-peer-urls", r.cfg.peerListenUrls,
		"--initial-advertise-peer-urls", r.cfg.peerAdvertiseUrls,
	)
	if r.cfg.etcdServerArgs != "" {
		extraArgs := strings.Fields(r.cfg.etcdServerArgs)
		etcdCmd.Args = append(etcdCmd.Args, extraArgs...)
	}
	fmt.Printf("Starting server %s: %+v\n", r.cfg.name, etcdCmd.Args)

	etcdCmd.Stdout = os.Stdout
	etcdCmd.Stderr = os.Stderr
	err := etcdCmd.Start()
	if err != nil {
		return err
	}
	interval := time.NewTicker(time.Millisecond * 500)
	defer interval.Stop()
	done := make(chan bool)
	go func() {
		time.Sleep(time.Minute * 2)
		close(done)
	}()
	for {
		select {
		case <-interval.C:
			err := r.client.SetEtcdVersionKeyValue(version)
			if err != nil {
				klog.Infof("Still waiting for etcd to start, current error: %v", err)
				// keep waiting
			} else {
				klog.Infof("Etcd on port %d is up.", r.cfg.port)
				r.cmd = etcdCmd
				return nil
			}
		case <-done:
			err = etcdCmd.Process.Kill()
			if err != nil {
				return fmt.Errorf("error killing etcd: %v", err)
			}
			return fmt.Errorf("timed out waiting for etcd on port %d", r.cfg.port)
		}
	}
}


Stop

// Stop terminates the etcd server process. If the etcd server process has not been started
// or is not still running, this returns an error.
func (r *EtcdMigrateServer) Stop() error {
	if r.cmd == nil {
		return fmt.Errorf("cannot stop EtcdMigrateServer that has not been started")
	}
	err := r.cmd.Process.Signal(os.Interrupt)
	if err != nil {
		return fmt.Errorf("error sending SIGINT to etcd for graceful shutdown: %v", err)
	}
	gracefulWait := time.Minute * 2
	stopped := make(chan bool)
	timedout := make(chan bool)
	go func() {
		time.Sleep(gracefulWait)
		close(timedout)
	}()
	go func() {
		select {
		case <-stopped:
			return
		case <-timedout:
			klog.Infof("etcd server has not terminated gracefully after %s, killing it.", gracefulWait)
			r.cmd.Process.Kill()
			return
		}
	}()
	err = r.cmd.Wait()
	close(stopped)
	if exiterr, ok := err.(*exec.ExitError); ok {
		klog.Infof("etcd server stopped (signal: %s)", exiterr.Error())
		// stopped
	} else if err != nil {
		return fmt.Errorf("error waiting for etcd to stop: %v", err)
	}
	klog.Infof("Stopped etcd server %s", r.cfg.name)
	return nil
}


MigrateIfNeeded

// MigrateIfNeeded upgrades or downgrades the etcd data directory to the given target version.
func (m *Migrator) MigrateIfNeeded(target *EtcdVersionPair) error {
	klog.Infof("Starting migration to %s", target)
	err := m.dataDirectory.Initialize(target)
	if err != nil {
		return fmt.Errorf("failed to initialize data directory %s: %v", m.dataDirectory.path, err)
	}

	var current *EtcdVersionPair
	vfExists, err := m.dataDirectory.versionFile.Exists()
	if err != nil {
		return err
	}
	if vfExists {
		current, err = m.dataDirectory.versionFile.Read()
		if err != nil {
			return err
		}
	} else {
		return fmt.Errorf("existing data directory '%s' is missing version.txt file, unable to migrate", m.dataDirectory.path)
	}

	for {
		klog.Infof("Converging current version '%s' to target version '%s'", current, target)
		currentNextMinorVersion := &EtcdVersion{Version: semver.Version{Major: current.version.Major, Minor: current.version.Minor + 1}}
		switch {
		case current.version.MajorMinorEquals(target.version) || currentNextMinorVersion.MajorMinorEquals(target.version):
			klog.Infof("current version '%s' equals or is one minor version previous of target version '%s' - migration complete", current, target)
			err = m.dataDirectory.versionFile.Write(target)
			if err != nil {
				return fmt.Errorf("failed to write version.txt to '%s': %v", m.dataDirectory.path, err)
			}
			return nil
		case current.storageVersion == storageEtcd2 && target.storageVersion == storageEtcd3:
			return fmt.Errorf("upgrading from etcd2 storage to etcd3 storage is not supported")
		case current.version.Major == 3 && target.version.Major == 2:
			return fmt.Errorf("downgrading from etcd 3.x to 2.x is not supported")
		case current.version.Major == target.version.Major && current.version.Minor < target.version.Minor:
			stepVersion := m.cfg.supportedVersions.NextVersionPair(current)
			klog.Infof("upgrading etcd from %s to %s", current, stepVersion)
			current, err = m.minorVersionUpgrade(current, stepVersion)
		case current.version.Major == 3 && target.version.Major == 3 && current.version.Minor > target.version.Minor:
			klog.Infof("rolling etcd back from %s to %s", current, target)
			current, err = m.rollbackEtcd3MinorVersion(current, target)
		}
		if err != nil {
			return err
		}
	}
}


func (m *Migrator) rollbackEtcd3MinorVersion(current *EtcdVersionPair, target *EtcdVersionPair) (*EtcdVersionPair, error) {
	if target.version.Minor != current.version.Minor-1 {
		return nil, fmt.Errorf("rollback from %s to %s not supported, only rollbacks to the previous minor version are supported", current.version, target.version)
	}

	klog.Infof("Performing etcd %s -> %s rollback", current.version, target.version)
	err := m.dataDirectory.Backup()
	if err != nil {
		return nil, err
	}

	snapshotFilename := fmt.Sprintf("%s.snapshot.db", m.dataDirectory.path)
	err = os.Remove(snapshotFilename)
	if err != nil && !os.IsNotExist(err) {
		return nil, fmt.Errorf("failed to clean snapshot file before rollback: %v", err)
	}

	// Start current version of etcd.
	runner := m.newServer()
	klog.Infof("Starting etcd version %s to capture rollback snapshot.", current.version)
	err = runner.Start(current.version)
	if err != nil {
		klog.Fatalf("Unable to automatically downgrade etcd: starting etcd version %s to capture rollback snapshot failed: %v", current.version, err)
		return nil, err
	}

	klog.Infof("Snapshotting etcd %s to %s", current.version, snapshotFilename)
	err = m.client.Snapshot(current.version, snapshotFilename)
	if err != nil {
		return nil, err
	}

	err = runner.Stop()
	if err != nil {
		return nil, err
	}

	klog.Info("Backing up data before rolling back")
	backupDir := fmt.Sprintf("%s.bak", m.dataDirectory)
	err = os.RemoveAll(backupDir)
	if err != nil {
		return nil, err
	}
	origInfo, err := os.Stat(m.dataDirectory.path)
	if err != nil {
		return nil, err
	}
	err = os.Rename(m.dataDirectory.path, backupDir)
	if err != nil {
		return nil, err
	}

	klog.Infof("Restoring etcd %s from %s", target.version, snapshotFilename)
	err = m.client.Restore(target.version, snapshotFilename)
	if err != nil {
		return nil, err
	}
	err = os.Chmod(m.dataDirectory.path, origInfo.Mode())
	if err != nil {
		return nil, err
	}

	return target, nil
}


minorVersionUpgrade

func (m *Migrator) minorVersionUpgrade(current *EtcdVersionPair, target *EtcdVersionPair) (*EtcdVersionPair, error) {
	runner := m.newServer()

	// Do the migration step, by just starting etcd in the target version.
	err := runner.Start(target.version)
	if err != nil {
		return nil, err
	}
	err = runner.Stop()
	return target, err
}


newServer

func (m *Migrator) newServer() *EtcdMigrateServer {
	return NewEtcdMigrateServer(m.cfg, m.client)
}


registerFlags

func registerFlags(flags *flag.FlagSet, opt *migrateOpts) {
	flags.StringVar(&opts.name, "name", "",
		"etcd cluster member name. If unset fallbacks to defaults to ETCD_NAME env, if unset defaults to etcd-<ETCD_HOSTNAME> env, if unset defaults to etcd-<HOSTNAME> env.")
	flags.Uint64Var(&opts.port, "port", 0,
		"etcd client port to use during migration operations. "+
			"This should be a different port than typically used by etcd to avoid clients accidentally connecting during upgrade/downgrade operations. "+
			"If unset default to 18629 or 18631 depending on <data-dir>.")
	flags.Uint64Var(&opts.peerPort, "peer-port", 0,
		"etcd peer port to use during migration operations. If unset defaults to 2380 or 2381 depending on <data-dir>.")
	flags.StringVar(&opts.peerListenUrls, "listen-peer-urls", "",
		"etcd --listen-peer-urls flag. If unset, fallbacks to LISTEN_PEER_URLS env and if unset defaults to http://localhost:<peer-port>.")
	flags.StringVar(&opts.peerAdvertiseUrls, "initial-advertise-peer-urls", "",
		"etcd --initial-advertise-peer-urls flag. If unset fallbacks to INITIAL_ADVERTISE_PEER_URLS env and if unset defaults to http://localhost:<peer-port>.")
	flags.StringVar(&opts.clientListenUrls, "listen-client-urls", "",
		"etcd --listen-client-urls flag. If unset, fallbacks to LISTEN_CLIENT_URLS env, and if unset defaults to http://127.0.0.1:<port>.")
	flags.StringVar(&opts.binDir, "bin-dir", "/usr/local/bin",
		"directory of etcd and etcdctl binaries, must contain etcd-<version> and etcdctl-<version> for each version listed in <bundled-versions>.")
	flags.StringVar(&opts.dataDir, "data-dir", "",
		"etcd data directory of etcd server to migrate. If unset fallbacks to DATA_DIRECTORY env.")
	flags.StringSliceVar(&opts.bundledVersions, "bundled-versions", supportedEtcdVersions,
		"comma separated list of etcd binary versions present under the bin-dir.")
	flags.StringVar(&opts.etcdDataPrefix, "etcd-data-prefix", "",
		"etcd key prefix under which all objects are kept. If unset fallbacks to ETCD_DATA_PREFIX env and if unset defaults to /registry.")
	flags.StringVar(&opts.ttlKeysDirectory, "ttl-keys-directory", "",
		"etcd key prefix under which all keys with TTLs are kept. Defaults to <etcd-data-prefix>/events")
	flags.StringVar(&opts.initialCluster, "initial-cluster", "",
		"comma separated list of name=endpoint pairs. If unset fallbacks to INITIAL_CLUSTER and if unset defaults to <etcd-name>=https://localhost:<peer-port>.")
	flags.StringVar(&opts.targetVersion, "target-version", "",
		"version of etcd to migrate to. Format must be <major>.<minor>.<patch>. If unset fallbacks to TARGET_VERSION env.")
	flags.StringVar(&opts.targetStorage, "target-storage", "",
		"storage version of etcd to migrate to, one of: etcd2, etcd3. If unset fallbacks to TARGET_STORAGE env.")
	flags.StringVar(&opts.etcdServerArgs, "etcd-server-extra-args", "",
		"additional etcd server args for starting etcd servers during migration steps, need to set TLS certs flags for multi-member clusters using mTLS for communication. "+
			"If unset fallbacks to ETCD_CREDS env.")
}


lookupEnv

func lookupEnv(env string) (string, error) {
	result, ok := os.LookupEnv(env)
	if !ok || len(result) == 0 {
		return result, fmt.Errorf("%s variable unset - expected failure", env)
	}
	return result, nil
}


fallbackToEnv

func fallbackToEnv(flag, env string) (string, error) {
	klog.Infof("--%s unset - falling back to %s variable", flag, env)
	return lookupEnv(env)
}


fallbackToEnvWithDefault

func fallbackToEnvWithDefault(flag, env, def string) string {
	if value, err := lookupEnv(env); err == nil {
		return value
	}
	klog.Warningf("%s variable for %s flag unset - defaulting to %s", env, flag, def)
	return def
}


defaultName

func defaultName() (string, error) {
	if etcdName, err := lookupEnv(etcdNameEnv); err == nil {
		return etcdName, nil
	}
	klog.Warningf("%s variable unset - falling back to etcd-%s variable", etcdNameEnv, etcdHostnameEnv)
	if etcdHostname, err := lookupEnv(etcdHostnameEnv); err == nil {
		return fmt.Sprintf("etcd-%s", etcdHostname), nil
	}
	klog.Warningf("%s variable unset - falling back to etcd-%s variable", etcdHostnameEnv, hostnameEnv)
	if hostname, err := lookupEnv(hostnameEnv); err == nil {
		return fmt.Sprintf("etcd-%s", hostname), nil
	}
	return "", fmt.Errorf("defaulting --name failed due to all ETCD_NAME, ETCD_HOSTNAME and HOSTNAME unset")
}


validateAndDefault

func (opts *migrateOpts) validateAndDefault() error {
	var err error

	if opts.name == "" {
		klog.Infof("--name unset - falling back to %s variable", etcdNameEnv)
		if opts.name, err = defaultName(); err != nil {
			return err
		}
	}

	if opts.dataDir == "" {
		if opts.dataDir, err = fallbackToEnv("data-dir", dataDirEnv); err != nil {
			return err
		}
	}

	etcdEventsRE := regexp.MustCompile("event")
	if opts.port == 0 {
		if etcdEventsRE.MatchString(opts.dataDir) {
			opts.port = 18631
		} else {
			opts.port = 18629
		}
		klog.Infof("--port unset - defaulting to %d", opts.port)
	}
	if opts.peerPort == 0 {
		if etcdEventsRE.MatchString(opts.dataDir) {
			opts.peerPort = 2381
		} else {
			opts.peerPort = 2380
		}
		klog.Infof("--peer-port unset - defaulting to %d", opts.peerPort)
	}

	if opts.initialCluster == "" {
		def := fmt.Sprintf(initialClusterFmt, opts.name, opts.peerPort)
		opts.initialCluster = fallbackToEnvWithDefault("initial-cluster", initialClusterEnv, def)
	}

	if opts.peerListenUrls == "" {
		def := fmt.Sprintf(peerListenUrlsFmt, opts.peerPort)
		opts.peerListenUrls = fallbackToEnvWithDefault("listen-peer-urls", peerListenUrlsEnv, def)
	}

	if opts.peerAdvertiseUrls == "" {
		def := fmt.Sprintf(peerAdvertiseUrlsFmt, opts.peerPort)
		opts.peerAdvertiseUrls = fallbackToEnvWithDefault("initial-advertise-peer-urls", peerAdvertiseUrlsEnv, def)
	}

	if opts.clientListenUrls == "" {
		def := fmt.Sprintf(clientListenURLFmt, opts.port)
		opts.clientListenUrls = fallbackToEnvWithDefault("listen-client-urls", clientListenURLsEnv, def)
	}

	if opts.targetVersion == "" {
		if opts.targetVersion, err = fallbackToEnv("target-version", targetVersionEnv); err != nil {
			return err
		}
	}

	if opts.targetStorage == "" {
		if opts.targetStorage, err = fallbackToEnv("target-storage", targetStorageEnv); err != nil {
			return err
		}
	}

	if opts.etcdDataPrefix == "" {
		opts.etcdDataPrefix = fallbackToEnvWithDefault("etcd-data-prefix", etcdDataPrefixEnv, etcdDataPrefixDefault)
	}

	if opts.ttlKeysDirectory == "" {
		opts.ttlKeysDirectory = fmt.Sprintf(ttlKeysDirectoryFmt, opts.etcdDataPrefix)
		klog.Infof("--ttl-keys-directory unset - defaulting to %s", opts.ttlKeysDirectory)
	}

	if opts.etcdServerArgs == "" {
		opts.etcdServerArgs = fallbackToEnvWithDefault("etcd-server-extra-args", etcdServerArgsEnv, "")
	}

	if opts.supportedVersions, err = ParseSupportedVersions(opts.bundledVersions); err != nil {
		return fmt.Errorf("failed to parse --bundled-versions: %v", err)
	}

	if err := validateBundledVersions(opts.supportedVersions, opts.binDir); err != nil {
		return fmt.Errorf("failed to validate that 'etcd-<version>' and 'etcdctl-<version>' binaries exist in --bin-dir '%s' for all --bundled-versions '%s': %v",
			opts.binDir, strings.Join(opts.bundledVersions, ","), err)
	}
	return nil
}


validateBundledVersions

// validateBundledVersions checks that 'etcd-<version>' and 'etcdctl-<version>' binaries exist in the binDir
// for each version in the bundledVersions list.
func validateBundledVersions(bundledVersions SupportedVersions, binDir string) error {
	for _, v := range bundledVersions {
		for _, binaryName := range []string{"etcd", "etcdctl"} {
			fn := filepath.Join(binDir, fmt.Sprintf("%s-%s", binaryName, v))
			if _, err := os.Stat(fn); err != nil {
				return fmt.Errorf("failed to validate '%s' binary exists for bundled-version '%s': %v", fn, v, err)
			}

		}
	}
	return nil
}


copyDirectory

func copyDirectory(source string, dest string) error {
	return fileutils.CopyDirectory(source, dest)
}


func copyDirectory(source string, dest string) error {
	return fmt.Errorf("no support for windows")
}


ParseEtcdVersion

// ParseEtcdVersion parses a SemVer string to an EtcdVersion.
func ParseEtcdVersion(s string) (*EtcdVersion, error) {
	v, err := semver.Make(s)
	if err != nil {
		return nil, err
	}
	return &EtcdVersion{v}, nil
}


MustParseEtcdVersion

// MustParseEtcdVersion parses a SemVer string to an EtcdVersion and panics if the parse fails.
func MustParseEtcdVersion(s string) *EtcdVersion {
	return &EtcdVersion{semver.MustParse(s)}
}


String

// String returns the version in SemVer string format.
func (v *EtcdVersion) String() string {
	return v.Version.String()
}


Equals

// Equals returns true if the versions are exactly equal.
func (v *EtcdVersion) Equals(o *EtcdVersion) bool {
	return v.Version.Equals(o.Version)
}


MajorMinorEquals

// MajorMinorEquals returns true if the major and minor parts of the versions are equal;
// if only patch versions differ, this returns true.
func (v *EtcdVersion) MajorMinorEquals(o *EtcdVersion) bool {
	return v.Major == o.Major && v.Minor == o.Minor
}


ParseEtcdStorageVersion

// ParseEtcdStorageVersion parses an etcd storage version string to an EtcdStorageVersion.
func ParseEtcdStorageVersion(s string) (EtcdStorageVersion, error) {
	switch s {
	case "etcd2":
		return storageEtcd2, nil
	case "etcd3":
		return storageEtcd3, nil
	default:
		return storageUnknown, fmt.Errorf("unrecognized storage version: %s", s)
	}
}


MustParseEtcdStorageVersion

// MustParseEtcdStorageVersion parses an etcd storage version string to an EtcdStorageVersion and
// panics if the parse fails.
func MustParseEtcdStorageVersion(s string) EtcdStorageVersion {
	version, err := ParseEtcdStorageVersion(s)
	if err != nil {
		panic(err)
	}
	return version
}


String

// String returns the text representation of the EtcdStorageVersion, 'etcd2' or 'etcd3'.
func (v EtcdStorageVersion) String() string {
	switch v {
	case storageEtcd2:
		return "etcd2"
	case storageEtcd3:
		return "etcd3"
	default:
		panic(fmt.Sprintf("enum value %d missing from EtcdStorageVersion String() function", v))
	}
}


ParseEtcdVersionPair

// ParseEtcdVersionPair parses a "<version>/<storage-version>" string to an EtcdVersionPair.
func ParseEtcdVersionPair(s string) (*EtcdVersionPair, error) {
	parts := strings.Split(s, "/")
	if len(parts) != 2 {
		return nil, fmt.Errorf("malformed version file, expected <major>.<minor>.<patch>/<storage> but got %s", s)
	}
	version, err := ParseEtcdVersion(parts[0])
	if err != nil {
		return nil, err
	}
	storageVersion, err := ParseEtcdStorageVersion(parts[1])
	if err != nil {
		return nil, err
	}
	return &EtcdVersionPair{version, storageVersion}, nil
}


String

// String returns "<version>/<storage-version>" string of the EtcdVersionPair.
func (vp *EtcdVersionPair) String() string {
	return fmt.Sprintf("%s/%s", vp.version, vp.storageVersion)
}


Equals

// Equals returns true if both the versions and storage versions are exactly equal.
func (vp *EtcdVersionPair) Equals(o *EtcdVersionPair) bool {
	return vp.version.Equals(o.version) && vp.storageVersion == o.storageVersion
}


NextVersion

// NextVersion returns the next supported version after the given current version, or nil if no
// next version exists.
func (sv SupportedVersions) NextVersion(current *EtcdVersion) *EtcdVersion {
	var nextVersion *EtcdVersion
	for i, supportedVersion := range sv {
		if current.MajorMinorEquals(supportedVersion) && len(sv) > i+1 {
			nextVersion = sv[i+1]
		}
	}
	return nextVersion
}


NextVersionPair

// NextVersionPair returns the next supported version after the given current version and infers
// the storage version from the major version part of the next version.
func (sv SupportedVersions) NextVersionPair(current *EtcdVersionPair) *EtcdVersionPair {
	nextVersion := sv.NextVersion(current.version)
	if nextVersion == nil {
		return nil
	}
	storageVersion := storageEtcd3
	if nextVersion.Major == 2 {
		storageVersion = storageEtcd2
	}
	return &EtcdVersionPair{version: nextVersion, storageVersion: storageVersion}
}


ParseSupportedVersions

// ParseSupportedVersions parses a list of etcd versions.
func ParseSupportedVersions(list []string) (SupportedVersions, error) {
	var err error
	versions := make(SupportedVersions, len(list))
	for i, v := range list {
		versions[i], err = ParseEtcdVersion(strings.TrimSpace(v))
		if err != nil {
			return nil, err
		}
	}
	return versions, nil
}


main

func main() {
	var errorCount int

	kubectl := cmd.NewKubectlCommand(cmd.KubectlOptions{IOStreams: genericclioptions.IOStreams{In: os.Stdin, Out: io.Discard, ErrOut: io.Discard}})
	errors := cmdsanity.RunCmdChecks(kubectl, cmdsanity.AllCmdChecks, []string{})
	for _, err := range errors {
		errorCount++
		fmt.Fprintf(os.Stderr, "     %d. %s\n", errorCount, err)
	}

	errors = cmdsanity.RunGlobalChecks(cmdsanity.AllGlobalChecks)
	for _, err := range errors {
		errorCount++
		fmt.Fprintf(os.Stderr, "     %d. %s\n", errorCount, err)
	}

	if errorCount > 0 {
		fmt.Fprintf(os.Stdout, "Found %d errors.\n", errorCount)
		os.Exit(1)
	}

	fmt.Fprintln(os.Stdout, "Congrats, CLI looks good!")
}


func main() {
	ccmOptions, err := options.NewCloudControllerManagerOptions()
	if err != nil {
		klog.Fatalf("unable to initialize command options: %v", err)
	}

	controllerInitializers := app.DefaultInitFuncConstructors
	// Here is an example to remove the controller which is not needed.
	// e.g. remove the cloud-node-lifecycle controller which current cloud provider does not need.
	//delete(controllerInitializers, "cloud-node-lifecycle")

	// Here is an example to add an controller(NodeIpamController) which will be used by cloud provider
	// generate nodeIPAMConfig. Here is an sample code.
	// If you do not need additional controller, please ignore.

	nodeIpamController := nodeIPAMController{}
	nodeIpamController.nodeIPAMControllerOptions.NodeIPAMControllerConfiguration = &nodeIpamController.nodeIPAMControllerConfiguration
	fss := cliflag.NamedFlagSets{}
	nodeIpamController.nodeIPAMControllerOptions.AddFlags(fss.FlagSet("nodeipam controller"))

	controllerInitializers["nodeipam"] = app.ControllerInitFuncConstructor{
		// "node-controller" is the shared identity of all node controllers, including node, node lifecycle, and node ipam.
		// See https://github.com/kubernetes/kubernetes/pull/72764#issuecomment-453300990 for more context.
		InitContext: app.ControllerInitContext{
			ClientName: "node-controller",
		},
		Constructor: nodeIpamController.StartNodeIpamControllerWrapper,
	}

	command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, fss, wait.NeverStop)
	code := cli.Run(command)
	os.Exit(code)
}


cloudInitializer

func cloudInitializer(config *cloudcontrollerconfig.CompletedConfig) cloudprovider.Interface {
	cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider
	// initialize cloud provider with the cloud provider name and config file provided
	cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile)
	if err != nil {
		klog.Fatalf("Cloud provider could not be initialized: %v", err)
	}
	if cloud == nil {
		klog.Fatalf("Cloud provider is nil")
	}

	if !cloud.HasClusterID() {
		if config.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
			klog.Warning("detected a cluster without a ClusterID.  A ClusterID will be required in the future.  Please tag your cluster to avoid any future issues")
		} else {
			klog.Fatalf("no ClusterID found.  A ClusterID is required for the cloud provider to function properly.  This check can be bypassed by setting the allow-untagged-cloud option")
		}
	}

	return cloud
}


StartNodeIpamControllerWrapper

func (nodeIpamController *nodeIPAMController) StartNodeIpamControllerWrapper(initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc {
	allErrors := nodeIpamController.nodeIPAMControllerOptions.Validate()
	if len(allErrors) > 0 {
		klog.Fatal("NodeIPAM controller values are not properly set.")
	}
	nodeIpamController.nodeIPAMControllerOptions.ApplyTo(&nodeIpamController.nodeIPAMControllerConfiguration)

	return func(ctx context.Context, controllerContext genericcontrollermanager.ControllerContext) (controller.Interface, bool, error) {
		return startNodeIpamController(ctx, initContext, completedConfig, nodeIpamController.nodeIPAMControllerConfiguration, controllerContext, cloud)
	}
}


startNodeIpamController

func startNodeIpamController(ctx context.Context, initContext app.ControllerInitContext, ccmConfig *cloudcontrollerconfig.CompletedConfig, nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration, controllerCtx genericcontrollermanager.ControllerContext, cloud cloudprovider.Interface) (controller.Interface, bool, error) {
	var serviceCIDR *net.IPNet
	var secondaryServiceCIDR *net.IPNet

	// should we start nodeIPAM
	if !ccmConfig.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs {
		return nil, false, nil
	}

	// Cannot run cloud ipam controller if cloud provider is nil (--cloud-provider not set or set to 'external')
	if cloud == nil && ccmConfig.ComponentConfig.KubeCloudShared.CIDRAllocatorType == string(ipam.CloudAllocatorType) {
		return nil, false, errors.New("--cidr-allocator-type is set to 'CloudAllocator' but cloud provider is not configured")
	}

	// failure: bad cidrs in config
	clusterCIDRs, dualStack, err := processCIDRs(ccmConfig.ComponentConfig.KubeCloudShared.ClusterCIDR)
	if err != nil {
		return nil, false, err
	}

	// failure: more than one cidr but they are not configured as dual stack
	if len(clusterCIDRs) > 1 && !dualStack {
		return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and they are not configured as dual stack (at least one from each IPFamily", len(clusterCIDRs))
	}

	// failure: more than cidrs is not allowed even with dual stack
	if len(clusterCIDRs) > 2 {
		return nil, false, fmt.Errorf("len of clusters is:%v > more than max allowed of 2", len(clusterCIDRs))
	}

	// service cidr processing
	if len(strings.TrimSpace(nodeIPAMConfig.ServiceCIDR)) != 0 {
		_, serviceCIDR, err = netutils.ParseCIDRSloppy(nodeIPAMConfig.ServiceCIDR)
		if err != nil {
			klog.ErrorS(err, "Unsuccessful parsing of service CIDR", "CIDR", nodeIPAMConfig.ServiceCIDR)
		}
	}

	if len(strings.TrimSpace(nodeIPAMConfig.SecondaryServiceCIDR)) != 0 {
		_, secondaryServiceCIDR, err = netutils.ParseCIDRSloppy(nodeIPAMConfig.SecondaryServiceCIDR)
		if err != nil {
			klog.ErrorS(err, "Unsuccessful parsing of service CIDR", "CIDR", nodeIPAMConfig.SecondaryServiceCIDR)
		}
	}

	// the following checks are triggered if both serviceCIDR and secondaryServiceCIDR are provided
	if serviceCIDR != nil && secondaryServiceCIDR != nil {
		// should be dual stack (from different IPFamilies)
		dualstackServiceCIDR, err := netutils.IsDualStackCIDRs([]*net.IPNet{serviceCIDR, secondaryServiceCIDR})
		if err != nil {
			return nil, false, fmt.Errorf("failed to perform dualstack check on serviceCIDR and secondaryServiceCIDR error:%v", err)
		}
		if !dualstackServiceCIDR {
			return nil, false, fmt.Errorf("serviceCIDR and secondaryServiceCIDR are not dualstack (from different IPfamiles)")
		}
	}

	nodeCIDRMaskSizes, err := setNodeCIDRMaskSizes(nodeIPAMConfig, clusterCIDRs)
	if err != nil {
		return nil, false, err
	}

	var clusterCIDRInformer v1alpha1.ClusterCIDRInformer
	if utilfeature.DefaultFeatureGate.Enabled(features.MultiCIDRRangeAllocator) {
		clusterCIDRInformer = controllerCtx.InformerFactory.Networking().V1alpha1().ClusterCIDRs()
	}
	nodeIpamController, err := nodeipamcontroller.NewNodeIpamController(
		ctx,
		controllerCtx.InformerFactory.Core().V1().Nodes(),
		clusterCIDRInformer,
		cloud,
		controllerCtx.ClientBuilder.ClientOrDie(initContext.ClientName),
		clusterCIDRs,
		serviceCIDR,
		secondaryServiceCIDR,
		nodeCIDRMaskSizes,
		ipam.CIDRAllocatorType(ccmConfig.ComponentConfig.KubeCloudShared.CIDRAllocatorType),
	)
	if err != nil {
		return nil, true, err
	}
	go nodeIpamController.Run(ctx)
	return nil, true, nil
}


processCIDRs

// processCIDRs is a helper function that works on a comma separated cidrs and returns
// a list of typed cidrs
// a flag if cidrs represents a dual stack
// error if failed to parse any of the cidrs
func processCIDRs(cidrsList string) ([]*net.IPNet, bool, error) {
	cidrsSplit := strings.Split(strings.TrimSpace(cidrsList), ",")

	cidrs, err := netutils.ParseCIDRs(cidrsSplit)
	if err != nil {
		return nil, false, err
	}

	// if cidrs has an error then the previous call will fail
	// safe to ignore error checking on next call
	dualstack, _ := netutils.IsDualStackCIDRs(cidrs)

	return cidrs, dualstack, nil
}


setNodeCIDRMaskSizes

// setNodeCIDRMaskSizes returns the IPv4 and IPv6 node cidr mask sizes to the value provided
// for --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 respectively. If value not provided,
// then it will return default IPv4 and IPv6 cidr mask sizes.
func setNodeCIDRMaskSizes(cfg nodeipamconfig.NodeIPAMControllerConfiguration, clusterCIDRs []*net.IPNet) ([]int, error) {

	sortedSizes := func(maskSizeIPv4, maskSizeIPv6 int) []int {
		nodeMaskCIDRs := make([]int, len(clusterCIDRs))

		for idx, clusterCIDR := range clusterCIDRs {
			if netutils.IsIPv6CIDR(clusterCIDR) {
				nodeMaskCIDRs[idx] = maskSizeIPv6
			} else {
				nodeMaskCIDRs[idx] = maskSizeIPv4
			}
		}
		return nodeMaskCIDRs
	}

	// --node-cidr-mask-size flag is incompatible with dual stack clusters.
	ipv4Mask, ipv6Mask := defaultNodeMaskCIDRIPv4, defaultNodeMaskCIDRIPv6
	isDualstack := len(clusterCIDRs) > 1

	// case one: cluster is dualstack (i.e, more than one cidr)
	if isDualstack {
		// if --node-cidr-mask-size then fail, user must configure the correct dual-stack mask sizes (or use default)
		if cfg.NodeCIDRMaskSize != 0 {
			return nil, errors.New("usage of --node-cidr-mask-size is not allowed with dual-stack clusters")
		}

		if cfg.NodeCIDRMaskSizeIPv4 != 0 {
			ipv4Mask = int(cfg.NodeCIDRMaskSizeIPv4)
		}
		if cfg.NodeCIDRMaskSizeIPv6 != 0 {
			ipv6Mask = int(cfg.NodeCIDRMaskSizeIPv6)
		}
		return sortedSizes(ipv4Mask, ipv6Mask), nil
	}

	maskConfigured := cfg.NodeCIDRMaskSize != 0
	maskV4Configured := cfg.NodeCIDRMaskSizeIPv4 != 0
	maskV6Configured := cfg.NodeCIDRMaskSizeIPv6 != 0
	isSingleStackIPv6 := netutils.IsIPv6CIDR(clusterCIDRs[0])

	// original flag is set
	if maskConfigured {
		// original mask flag is still the main reference.
		if maskV4Configured || maskV6Configured {
			return nil, errors.New("usage of --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 is not allowed if --node-cidr-mask-size is set. For dual-stack clusters please unset it and use IPFamily specific flags")
		}

		mask := int(cfg.NodeCIDRMaskSize)
		return sortedSizes(mask, mask), nil
	}

	if maskV4Configured {
		if isSingleStackIPv6 {
			return nil, errors.New("usage of --node-cidr-mask-size-ipv4 is not allowed for a single-stack IPv6 cluster")
		}

		ipv4Mask = int(cfg.NodeCIDRMaskSizeIPv4)
	}

	// !maskV4Configured && !maskConfigured && maskV6Configured
	if maskV6Configured {
		if !isSingleStackIPv6 {
			return nil, errors.New("usage of --node-cidr-mask-size-ipv6 is not allowed for a single-stack IPv4 cluster")
		}

		ipv6Mask = int(cfg.NodeCIDRMaskSizeIPv6)
	}
	return sortedSizes(ipv4Mask, ipv6Mask), nil
}


main

func main() {
	flag.Parse()

	args := flag.Args()

	if len(args) != 1 {
		log.Fatalf("usage: dependencycheck <json-dep-file> (e.g. 'go list -mod=vendor -test -deps -json ./vendor/...')")
	}
	if *restrict == "" {
		log.Fatalf("Must specify restricted regex pattern")
	}
	depsPattern, err := regexp.Compile(*restrict)
	if err != nil {
		log.Fatalf("Error compiling restricted dependencies regex: %v", err)
	}
	var excludePattern *regexp.Regexp
	if *exclude != "" {
		excludePattern, err = regexp.Compile(*exclude)
		if err != nil {
			log.Fatalf("Error compiling excluded package regex: %v", err)
		}
	}
	b, err := os.ReadFile(args[0])
	if err != nil {
		log.Fatalf("Error reading dependencies file: %v", err)
	}

	packages := []goPackage{}
	decoder := json.NewDecoder(bytes.NewBuffer(b))
	for {
		pkg := goPackage{}
		if err := decoder.Decode(&pkg); err != nil {
			if err == io.EOF {
				break
			}
			log.Fatalf("Error unmarshaling dependencies file: %v", err)
		}
		packages = append(packages, pkg)
	}

	violations := map[string][]string{}
	for _, p := range packages {
		if excludePattern != nil && excludePattern.MatchString(p.ImportPath) {
			continue
		}
		importViolations := []string{}
		allImports := []string{}
		allImports = append(allImports, p.Imports...)
		allImports = append(allImports, p.TestImports...)
		allImports = append(allImports, p.XTestImports...)
		for _, i := range allImports {
			if depsPattern.MatchString(i) {
				importViolations = append(importViolations, i)
			}
		}
		if len(importViolations) > 0 {
			violations[p.ImportPath] = importViolations
		}
	}

	if len(violations) > 0 {
		for k, v := range violations {
			fmt.Printf("Found dependency violations in package %s:\n", k)
			for _, a := range v {
				fmt.Println("--> " + a)
			}
		}
		log.Fatal("Found restricted dependency violations in packages")
	}
}


runCommand

// runCommand runs the cmd and returns the combined stdout and stderr, or an
// error if the command failed.
func runCommand(cmd ...string) (string, error) {
	output, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("failed to run %q: %s (%s)", strings.Join(cmd, " "), err, output)
	}
	return string(output), nil
}


readFile

func readFile(path string) (string, error) {
	content, err := os.ReadFile(path)
	// Convert []byte to string and print to screen
	return string(content), err
}


moduleInSlice

func moduleInSlice(a module, list []module, matchVersion bool) bool {
	for _, b := range list {
		if b == a {
			return true
		}
		if !matchVersion && b.name == a.name {
			return true
		}
	}
	return false
}


convertToMap

// converts `go mod graph` output modStr into a map of from->[]to references and the main module
func convertToMap(modStr string) ([]module, map[module][]module) {
	var (
		mainModulesList = []module{}
		mainModules     = map[module]bool{}
	)
	modMap := make(map[module][]module)
	for _, line := range strings.Split(modStr, "\n") {
		if len(line) == 0 {
			continue
		}
		deps := strings.Split(line, " ")
		if len(deps) == 2 {
			first := parseModule(deps[0])
			second := parseModule(deps[1])
			if first.version == "" || first.version == "v0.0.0" {
				if !mainModules[first] {
					mainModules[first] = true
					mainModulesList = append(mainModulesList, first)
				}
			}
			modMap[first] = append(modMap[first], second)
		} else {
			// skip invalid line
			log.Printf("!!!invalid line in mod.graph: %s", line)
			continue
		}
	}
	return mainModulesList, modMap
}


difference

// difference returns a-b and b-a as sorted lists
func difference(a, b []string) ([]string, []string) {
	aMinusB := map[string]bool{}
	bMinusA := map[string]bool{}
	for _, dependency := range a {
		aMinusB[dependency] = true
	}
	for _, dependency := range b {
		if _, found := aMinusB[dependency]; found {
			delete(aMinusB, dependency)
		} else {
			bMinusA[dependency] = true
		}
	}
	aMinusBList := []string{}
	bMinusAList := []string{}
	for dependency := range aMinusB {
		aMinusBList = append(aMinusBList, dependency)
	}
	for dependency := range bMinusA {
		bMinusAList = append(bMinusAList, dependency)
	}
	sort.Strings(aMinusBList)
	sort.Strings(bMinusAList)
	return aMinusBList, bMinusAList
}


String

func (m module) String() string {
	if len(m.version) == 0 {
		return m.name
	}
	return m.name + "@" + m.version
}


parseModule

func parseModule(s string) module {
	if !strings.Contains(s, "@") {
		return module{name: s}
	}
	parts := strings.SplitN(s, "@", 2)
	return module{name: parts[0], version: parts[1]}
}


main

// option1: dependencyverifier dependencies.json
// it will run `go mod graph` and check it.
// option2: dependencyverifier dependencies.json mod.graph
// it will check the specified mod graph result file.
func main() {
	var modeGraphStr string
	var err error
	if len(os.Args) == 2 {
		// run `go mod graph`
		modeGraphStr, err = runCommand("go", "mod", "graph")
		if err != nil {
			log.Fatalf("Error running 'go mod graph': %s", err)
		}
	} else if len(os.Args) == 3 {
		modGraphFile := string(os.Args[2])
		modeGraphStr, err = readFile(modGraphFile)
		// read file, such as `mod.graph`
		if err != nil {
			log.Fatalf("Error reading mod file %s: %s", modGraphFile, err)
		}
	} else {
		log.Fatalf("Usage: %s dependencies.json {mod.graph}", os.Args[0])
	}

	dependenciesJSONPath := string(os.Args[1])
	dependencies, err := readFile(dependenciesJSONPath)
	if err != nil {
		log.Fatalf("Error reading dependencies file %s: %s", dependencies, err)
	}

	// load Unwanted from json
	configFromFile := &Unwanted{}
	decoder := json.NewDecoder(bytes.NewBuffer([]byte(dependencies)))
	decoder.DisallowUnknownFields()
	if err := decoder.Decode(configFromFile); err != nil {
		log.Fatalf("Error reading dependencies file %s: %s", dependenciesJSONPath, err)
	}

	// convert from `go mod graph` to main module and map of from->[]to references
	mainModules, moduleGraph := convertToMap(modeGraphStr)

	// gather the effective versions by looking at the versions required by the main modules
	effectiveVersions := map[string]module{}
	for _, mainModule := range mainModules {
		for _, override := range moduleGraph[mainModule] {
			if _, ok := effectiveVersions[override.name]; !ok {
				effectiveVersions[override.name] = override
			}
		}
	}

	unwantedToReferencers := map[string][]module{}
	for _, mainModule := range mainModules {
		// visit to find unwanted modules still referenced from the main module
		visit(func(m module, via []module) {
			if _, unwanted := configFromFile.Spec.UnwantedModules[m.name]; unwanted {
				// this is unwanted, store what is referencing it
				referencer := via[len(via)-1]
				if !moduleInSlice(referencer, unwantedToReferencers[m.name], false) {
					// // uncomment to get a detailed tree of the path that referenced the unwanted dependency
					//
					// i := 0
					// for _, v := range via {
					// 	if v.version != "" && v.version != "v0.0.0" {
					// 		fmt.Println(strings.Repeat("  ", i), v)
					// 		i++
					// 	}
					// }
					// if i > 0 {
					// 	fmt.Println(strings.Repeat("  ", i+1), m)
					// 	fmt.Println()
					// }
					unwantedToReferencers[m.name] = append(unwantedToReferencers[m.name], referencer)
				}
			}
		}, mainModule, moduleGraph, effectiveVersions)
	}

	config := &Unwanted{}
	config.Spec.UnwantedModules = configFromFile.Spec.UnwantedModules
	for unwanted := range unwantedToReferencers {
		if config.Status.UnwantedReferences == nil {
			config.Status.UnwantedReferences = map[string][]string{}
		}
		sort.Slice(unwantedToReferencers[unwanted], func(i, j int) bool {
			ri := unwantedToReferencers[unwanted][i]
			rj := unwantedToReferencers[unwanted][j]
			if ri.name != rj.name {
				return ri.name < rj.name
			}
			return ri.version < rj.version
		})
		for _, referencer := range unwantedToReferencers[unwanted] {
			// make sure any reference at all shows up as a non-nil status
			if config.Status.UnwantedReferences == nil {
				config.Status.UnwantedReferences[unwanted] = []string{}
			}
			// record specific names of versioned referents
			if referencer.version != "" && referencer.version != "v0.0.0" {
				config.Status.UnwantedReferences[unwanted] = append(config.Status.UnwantedReferences[unwanted], referencer.name)
			}
		}
	}

	needUpdate := false

	// Compare unwanted list from unwanted-dependencies.json with current status from `go mod graph`
	expected, err := json.MarshalIndent(configFromFile.Status, "", "  ")
	if err != nil {
		log.Fatal(err)
	}
	actual, err := json.MarshalIndent(config.Status, "", "  ")
	if err != nil {
		log.Fatal(err)
	}
	if !bytes.Equal(expected, actual) {
		log.Printf("Expected status of\n%s", string(expected))
		log.Printf("Got status of\n%s", string(actual))
	}
	for expectedRef, expectedFrom := range configFromFile.Status.UnwantedReferences {
		actualFrom, ok := config.Status.UnwantedReferences[expectedRef]
		if !ok {
			// disappeared entirely
			log.Printf("Good news! Unwanted dependency %q is no longer referenced. Remove status.unwantedReferences[%q] in %s to ensure it doesn't get reintroduced.", expectedRef, expectedRef, dependenciesJSONPath)
			needUpdate = true
			continue
		}
		removedReferences, unwantedReferences := difference(expectedFrom, actualFrom)
		if len(removedReferences) > 0 {
			log.Printf("Good news! Unwanted module %q dropped the following dependants:", expectedRef)
			for _, reference := range removedReferences {
				log.Printf("   %s", reference)
			}
			log.Printf("!!! Remove those from status.unwantedReferences[%q] in %s to ensure they don't get reintroduced.", expectedRef, dependenciesJSONPath)
			needUpdate = true
		}
		if len(unwantedReferences) > 0 {
			log.Printf("Unwanted module %q marked in %s is referenced by new dependants:", expectedRef, dependenciesJSONPath)
			for _, reference := range unwantedReferences {
				log.Printf("   %s", reference)
			}
			log.Printf("!!! Avoid updating referencing modules to versions that reintroduce use of unwanted dependencies\n")
			needUpdate = true
		}
	}
	for actualRef, actualFrom := range config.Status.UnwantedReferences {
		if _, expected := configFromFile.Status.UnwantedReferences[actualRef]; expected {
			// expected, already ensured referencers were equal in the first loop
			continue
		}
		log.Printf("Unwanted module %q marked in %s is referenced", actualRef, dependenciesJSONPath)
		for _, reference := range actualFrom {
			log.Printf("   %s", reference)
		}
		log.Printf("!!! Avoid updating referencing modules to versions that reintroduce use of unwanted dependencies\n")
		needUpdate = true
	}

	if needUpdate {
		os.Exit(1)
	}
}


visit

func visit(visitor func(m module, via []module), main module, references map[module][]module, effectiveVersions map[string]module) {
	doVisit(visitor, main, nil, map[module]bool{}, references, effectiveVersions)
}


doVisit

func doVisit(visitor func(m module, via []module), from module, via []module, visited map[module]bool, references map[module][]module, effectiveVersions map[string]module) {
	visitor(from, via)
	via = append(via, from)
	if visited[from] {
		return
	}
	for _, to := range references[from] {
		// switch to the effective version of this dependency
		if override, ok := effectiveVersions[to.name]; ok {
			to = override
		}
		// recurse unless we've already visited this module in this traversal
		if !moduleInSlice(to, via, false) {
			doVisit(visitor, to, via, visited, references, effectiveVersions)
		}
	}
	visited[from] = true
}


main

func main() {
	flag.Parse()

	if *typeSrc == "" {
		klog.Fatalf("Please define -s flag as it is the api type file")
	}

	docsForTypes := kruntime.ParseDocumentationFrom(*typeSrc)
	rc := false

	for _, ks := range docsForTypes {
		typesMap := make(kubeTypesMap)

		for _, p := range ks[1:] {
			// skip the field with no tag name
			if p.Name != "" {
				typesMap[strings.ToLower(p.Name)] = p
			}
		}

		structName := ks[0].Name

		rc = checkFieldNameAndDoc(structName, "", ks[0].Doc, typesMap)
		for _, p := range ks[1:] {
			rc = checkFieldNameAndDoc(structName, p.Name, p.Doc, typesMap)
		}
	}

	if rc {
		os.Exit(1)
	}
}


checkFieldNameAndDoc

func checkFieldNameAndDoc(structName, fieldName, doc string, typesMap kubeTypesMap) bool {
	rc := false
	visited := sets.Set[string]{}

	// The rule is:
	// 1. Get all back-tick quoted names in the doc
	// 2. Skip the name which is already found mismatched.
	// 3. Skip the name whose lowercase is different from the lowercase of tag names,
	//    because some docs use back-tick to quote field value or nil
	// 4. Check if the name is different from its tag name

	// TODO: a manual pass adding back-ticks to the doc strings, then update the linter to
	// check the existence of back-ticks
	nameGroups := re.FindAllStringSubmatch(doc, -1)
	for _, nameGroup := range nameGroups {
		name := nameGroup[1]
		if visited.Has(name) {
			continue
		}
		if p, ok := typesMap[strings.ToLower(name)]; ok && p.Name != name {
			rc = true
			visited.Insert(name)

			fmt.Fprintf(os.Stderr, "Error: doc for %s", structName)
			if fieldName != "" {
				fmt.Fprintf(os.Stderr, ".%s", fieldName)
			}

			fmt.Fprintf(os.Stderr, " contains: %s, which should be: %s\n", name, p.Name)
		}
	}

	return rc
}


main

func main() {
	// use os.Args instead of "flags" because "flags" will mess up the man pages!
	path := "docs/"
	if len(os.Args) == 2 {
		path = os.Args[1]
	} else if len(os.Args) > 2 {
		fmt.Fprintf(os.Stderr, "usage: %s [output directory]\n", os.Args[0])
		os.Exit(1)
	}

	outDir, err := genutils.OutDir(path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to get output directory: %v\n", err)
		os.Exit(1)
	}

	// Set environment variables used by kubectl so the output is consistent,
	// regardless of where we run.
	os.Setenv("HOME", "/home/username")
	kubectl := cmd.NewKubectlCommand(cmd.KubectlOptions{IOStreams: genericclioptions.IOStreams{In: bytes.NewReader(nil), Out: io.Discard, ErrOut: io.Discard}})
	doc.GenMarkdownTree(kubectl, outDir)
}


func main() {
	// use os.Args instead of "flags" because "flags" will mess up the man pages!
	path := ""
	module := ""
	if len(os.Args) == 3 {
		path = os.Args[1]
		module = os.Args[2]
	} else {
		fmt.Fprintf(os.Stderr, "usage: %s [output directory] [module] \n", os.Args[0])
		os.Exit(1)
	}

	outDir, err := genutils.OutDir(path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to get output directory: %v\n", err)
		os.Exit(1)
	}

	switch module {
	case "kube-apiserver":
		// generate docs for kube-apiserver
		apiserver := apiservapp.NewAPIServerCommand()
		doc.GenMarkdownTree(apiserver, outDir)
	case "kube-controller-manager":
		// generate docs for kube-controller-manager
		controllermanager := cmapp.NewControllerManagerCommand()
		doc.GenMarkdownTree(controllermanager, outDir)
	case "kube-proxy":
		// generate docs for kube-proxy
		proxy := proxyapp.NewProxyCommand()
		doc.GenMarkdownTree(proxy, outDir)
	case "kube-scheduler":
		// generate docs for kube-scheduler
		scheduler := schapp.NewSchedulerCommand()
		doc.GenMarkdownTree(scheduler, outDir)
	case "kubelet":
		// generate docs for kubelet
		kubelet := kubeletapp.NewKubeletCommand()
		doc.GenMarkdownTree(kubelet, outDir)
	case "kubeadm":
		// resets global flags created by kubelet or other commands e.g.
		// --azure-container-registry-config from pkg/credentialprovider/azure
		// --version pkg/version/verflag
		pflag.CommandLine = pflag.NewFlagSet(os.Args[0], pflag.ExitOnError)

		// generate docs for kubeadm
		kubeadm := kubeadmapp.NewKubeadmCommand(bytes.NewReader(nil), io.Discard, io.Discard)
		doc.GenMarkdownTree(kubeadm, outDir)

		// cleanup generated code for usage as include in the website
		MarkdownPostProcessing(kubeadm, outDir, cleanupForInclude)
	default:
		fmt.Fprintf(os.Stderr, "Module %s is not supported", module)
		os.Exit(1)
	}
}


MarkdownPostProcessing

// MarkdownPostProcessing goes though the generated files
func MarkdownPostProcessing(cmd *cobra.Command, dir string, processor func(string) string) error {
	for _, c := range cmd.Commands() {
		if !c.IsAvailableCommand() || c.IsAdditionalHelpTopicCommand() {
			continue
		}
		if err := MarkdownPostProcessing(c, dir, processor); err != nil {
			return err
		}
	}

	basename := strings.Replace(cmd.CommandPath(), " ", "_", -1) + ".md"
	filename := filepath.Join(dir, basename)

	markdownBytes, err := os.ReadFile(filename)
	if err != nil {
		return err
	}

	processedMarkDown := processor(string(markdownBytes))

	return os.WriteFile(filename, []byte(processedMarkDown), 0644)
}


cleanupForInclude

// cleanupForInclude parts of markdown that will make difficult to use it as include in the website:
// - The title of the document (this allow more flexibility for include, e.g. include in tabs)
// - The sections see also, that assumes file will be used as a main page
func cleanupForInclude(md string) string {
	lines := strings.Split(md, "\n")

	cleanMd := ""
	for i, line := range lines {
		if i == 0 {
			continue
		}
		if line == "### SEE ALSO" {
			break
		}

		cleanMd += line
		if i < len(lines)-1 {
			cleanMd += "\n"
		}
	}

	return cleanMd
}


main

func main() {
	// use os.Args instead of "flags" because "flags" will mess up the man pages!
	path := "docs/man/man1"
	module := ""
	if len(os.Args) == 3 {
		path = os.Args[1]
		module = os.Args[2]
	} else {
		fmt.Fprintf(os.Stderr, "usage: %s [output directory] [module] \n", os.Args[0])
		os.Exit(1)
	}

	outDir, err := genutils.OutDir(path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to get output directory: %v\n", err)
		os.Exit(1)
	}

	// Set environment variables used by command so the output is consistent,
	// regardless of where we run.
	os.Setenv("HOME", "/home/username")

	switch module {
	case "kube-apiserver":
		// generate manpage for kube-apiserver
		apiserver := apiservapp.NewAPIServerCommand()
		genMarkdown(apiserver, "", outDir)
		for _, c := range apiserver.Commands() {
			genMarkdown(c, "kube-apiserver", outDir)
		}
	case "kube-controller-manager":
		// generate manpage for kube-controller-manager
		controllermanager := cmapp.NewControllerManagerCommand()
		genMarkdown(controllermanager, "", outDir)
		for _, c := range controllermanager.Commands() {
			genMarkdown(c, "kube-controller-manager", outDir)
		}
	case "kube-proxy":
		// generate manpage for kube-proxy
		proxy := proxyapp.NewProxyCommand()
		genMarkdown(proxy, "", outDir)
		for _, c := range proxy.Commands() {
			genMarkdown(c, "kube-proxy", outDir)
		}
	case "kube-scheduler":
		// generate manpage for kube-scheduler
		scheduler := schapp.NewSchedulerCommand()
		genMarkdown(scheduler, "", outDir)
		for _, c := range scheduler.Commands() {
			genMarkdown(c, "kube-scheduler", outDir)
		}
	case "kubelet":
		// generate manpage for kubelet
		kubelet := kubeletapp.NewKubeletCommand()
		genMarkdown(kubelet, "", outDir)
		for _, c := range kubelet.Commands() {
			genMarkdown(c, "kubelet", outDir)
		}
	case "kubectl":
		// generate manpage for kubectl
		kubectl := kubectlcmd.NewKubectlCommand(kubectlcmd.KubectlOptions{IOStreams: genericclioptions.IOStreams{In: bytes.NewReader(nil), Out: io.Discard, ErrOut: io.Discard}})
		genMarkdown(kubectl, "", outDir)
		for _, c := range kubectl.Commands() {
			genMarkdown(c, "kubectl", outDir)
		}
	case "kubeadm":
		// generate manpage for kubeadm
		kubeadm := kubeadmapp.NewKubeadmCommand(bytes.NewReader(nil), io.Discard, io.Discard)
		genMarkdown(kubeadm, "", outDir)
		for _, c := range kubeadm.Commands() {
			genMarkdown(c, "kubeadm", outDir)
		}
	default:
		fmt.Fprintf(os.Stderr, "Module %s is not supported", module)
		os.Exit(1)
	}
}


preamble

func preamble(out *bytes.Buffer, name, short, long string) {
	out.WriteString(`% KUBERNETES(1) kubernetes User Manuals
% Eric Paris
% Jan 2015
# NAME
`)
	fmt.Fprintf(out, "%s \\- %s\n\n", name, short)
	fmt.Fprintf(out, "# SYNOPSIS\n")
	fmt.Fprintf(out, "**%s** [OPTIONS]\n\n", name)
	fmt.Fprintf(out, "# DESCRIPTION\n")
	fmt.Fprintf(out, "%s\n\n", long)
}


printFlags

func printFlags(out *bytes.Buffer, flags *pflag.FlagSet) {
	flags.VisitAll(func(flag *pflag.Flag) {
		format := "**--%s**=%s\n\t%s\n\n"
		if flag.Value.Type() == "string" {
			// put quotes on the value
			format = "**--%s**=%q\n\t%s\n\n"
		}

		// Todo, when we mark a shorthand is deprecated, but specify an empty message.
		// The flag.ShorthandDeprecated is empty as the shorthand is deprecated.
		// Using len(flag.ShorthandDeprecated) > 0 can't handle this, others are ok.
		if !(len(flag.ShorthandDeprecated) > 0) && len(flag.Shorthand) > 0 {
			format = "**-%s**, " + format
			fmt.Fprintf(out, format, flag.Shorthand, flag.Name, flag.DefValue, flag.Usage)
		} else {
			fmt.Fprintf(out, format, flag.Name, flag.DefValue, flag.Usage)
		}
	})
}


printOptions

func printOptions(out *bytes.Buffer, command *cobra.Command) {
	flags := command.NonInheritedFlags()
	if flags.HasFlags() {
		fmt.Fprintf(out, "# OPTIONS\n")
		printFlags(out, flags)
		fmt.Fprintf(out, "\n")
	}
	flags = command.InheritedFlags()
	if flags.HasFlags() {
		fmt.Fprintf(out, "# OPTIONS INHERITED FROM PARENT COMMANDS\n")
		printFlags(out, flags)
		fmt.Fprintf(out, "\n")
	}
}


genMarkdown

func genMarkdown(command *cobra.Command, parent, docsDir string) {
	dparent := strings.Replace(parent, " ", "-", -1)
	name := command.Name()
	dname := name
	if len(parent) > 0 {
		dname = dparent + "-" + name
		name = parent + " " + name
	}

	out := new(bytes.Buffer)
	short := command.Short
	long := command.Long
	if len(long) == 0 {
		long = short
	}

	preamble(out, name, short, long)
	printOptions(out, command)

	if len(command.Example) > 0 {
		fmt.Fprintf(out, "# EXAMPLE\n")
		fmt.Fprintf(out, "```\n%s\n```\n", command.Example)
	}

	if len(command.Commands()) > 0 || len(parent) > 0 {
		fmt.Fprintf(out, "# SEE ALSO\n")
		if len(parent) > 0 {
			fmt.Fprintf(out, "**%s(1)**, ", dparent)
		}
		for _, c := range command.Commands() {
			fmt.Fprintf(out, "**%s-%s(1)**, ", dname, c.Name())
			genMarkdown(c, name, docsDir)
		}
		fmt.Fprintf(out, "\n")
	}

	out.WriteString(`
# HISTORY
January 2015, Originally compiled by Eric Paris (eparis at redhat dot com) based on the kubernetes source material, but hopefully they have been automatically generated since!
`)

	final := mangen.Render(out.Bytes())

	filename := docsDir + dname + ".1"
	outFile, err := os.Create(filename)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	defer outFile.Close()
	_, err = outFile.Write(final)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

}


main

func main() {
	flag.Parse()

	if *typeSrc == "" {
		klog.Fatalf("Please define -s flag as it is the source file")
	}

	var funcOut io.Writer
	if *functionDest == "-" {
		funcOut = os.Stdout
	} else {
		file, err := os.Create(*functionDest)
		if err != nil {
			klog.Fatalf("Couldn't open %v: %v", *functionDest, err)
		}
		defer file.Close()
		funcOut = file
	}

	docsForTypes := kruntime.ParseDocumentationFrom(*typeSrc)

	if *verify {
		rc, err := kruntime.VerifySwaggerDocsExist(docsForTypes, funcOut)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error in verification process: %s\n", err)
		}
		os.Exit(rc)
	}

	if len(docsForTypes) > 0 {
		if err := kruntime.WriteSwaggerDocFunc(docsForTypes, funcOut); err != nil {
			fmt.Fprintf(os.Stderr, "Error when writing swagger documentation functions: %s\n", err)
			os.Exit(-1)
		}
	}
}


OutDir

// OutDir creates the absolute path name from path and checks path exists.
// Returns absolute path including trailing '/' or error if path does not exist.
func OutDir(path string) (string, error) {
	outDir, err := filepath.Abs(path)
	if err != nil {
		return "", err
	}

	stat, err := os.Stat(outDir)
	if err != nil {
		return "", err
	}

	if !stat.IsDir() {
		return "", fmt.Errorf("output directory %s is not a directory", outDir)
	}
	outDir = outDir + "/"
	return outDir, nil
}


main

func main() {
	path := "docs/yaml/kubectl"
	if len(os.Args) == 2 {
		path = os.Args[1]
	} else if len(os.Args) > 2 {
		fmt.Fprintf(os.Stderr, "usage: %s [output directory]\n", os.Args[0])
		os.Exit(1)
	}

	outDir, err := genutils.OutDir(path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to get output directory: %v\n", err)
		os.Exit(1)
	}

	// Set environment variables used by kubectl so the output is consistent,
	// regardless of where we run.
	os.Setenv("HOME", "/home/username")
	kubectl := cmd.NewKubectlCommand(cmd.KubectlOptions{IOStreams: genericclioptions.IOStreams{In: bytes.NewReader(nil), Out: io.Discard, ErrOut: io.Discard}})
	genYaml(kubectl, "", outDir)
	for _, c := range kubectl.Commands() {
		genYaml(c, "kubectl", outDir)
	}
}


forceMultiLine

// Temporary workaround for yaml lib generating incorrect yaml with long strings
// that do not contain \n.
func forceMultiLine(s string) string {
	if len(s) > 60 && !strings.Contains(s, "\n") {
		s = s + "\n"
	}
	return s
}


genFlagResult

func genFlagResult(flags *pflag.FlagSet) []cmdOption {
	result := []cmdOption{}

	flags.VisitAll(func(flag *pflag.Flag) {
		// Todo, when we mark a shorthand is deprecated, but specify an empty message.
		// The flag.ShorthandDeprecated is empty as the shorthand is deprecated.
		// Using len(flag.ShorthandDeprecated) > 0 can't handle this, others are ok.
		if !(len(flag.ShorthandDeprecated) > 0) && len(flag.Shorthand) > 0 {
			opt := cmdOption{
				flag.Name,
				flag.Shorthand,
				flag.DefValue,
				forceMultiLine(flag.Usage),
			}
			result = append(result, opt)
		} else {
			opt := cmdOption{
				Name:         flag.Name,
				DefaultValue: forceMultiLine(flag.DefValue),
				Usage:        forceMultiLine(flag.Usage),
			}
			result = append(result, opt)
		}
	})

	return result
}


genYaml

func genYaml(command *cobra.Command, parent, docsDir string) {
	doc := cmdDoc{}

	doc.Name = command.Name()
	doc.Synopsis = forceMultiLine(command.Short)
	doc.Description = forceMultiLine(command.Long)

	flags := command.NonInheritedFlags()
	if flags.HasFlags() {
		doc.Options = genFlagResult(flags)
	}
	flags = command.InheritedFlags()
	if flags.HasFlags() {
		doc.InheritedOptions = genFlagResult(flags)
	}

	if len(command.Example) > 0 {
		doc.Example = command.Example
	}

	if len(command.Commands()) > 0 || len(parent) > 0 {
		result := []string{}
		if len(parent) > 0 {
			result = append(result, parent)
		}
		for _, c := range command.Commands() {
			result = append(result, c.Name())
		}
		doc.SeeAlso = result
	}

	final, err := yaml.Marshal(&doc)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	var filename string

	if parent == "" {
		filename = docsDir + doc.Name + ".yaml"
	} else {
		filename = docsDir + parent + "_" + doc.Name + ".yaml"
	}

	outFile, err := os.Create(filename)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	defer outFile.Close()
	_, err = outFile.Write(final)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}


ForbiddenImportsFor

// ForbiddenImportsFor determines all of the forbidden
// imports for a package given the import restrictions
func (i *ImportRestriction) ForbiddenImportsFor(pkg Package) ([]string, error) {
	if restricted, err := i.isRestrictedDir(pkg.Dir); err != nil {
		return []string{}, err
	} else if !restricted {
		return []string{}, nil
	}

	return i.forbiddenImportsFor(pkg), nil
}


isRestrictedDir

// isRestrictedDir determines if the source directory has
// any restrictions placed on it by this configuration.
// A path will be restricted if:
//   - it falls under the base import path
//   - it does not fall under any of the ignored sub-trees
func (i *ImportRestriction) isRestrictedDir(dir string) (bool, error) {
	if under, err := isPathUnder(i.BaseDir, dir); err != nil {
		return false, err
	} else if !under {
		return false, nil
	}

	for _, ignored := range i.IgnoredSubTrees {
		if under, err := isPathUnder(ignored, dir); err != nil {
			return false, err
		} else if under {
			return false, nil
		}
	}

	return true, nil
}


isPathUnder

// isPathUnder determines if path is under base
func isPathUnder(base, path string) (bool, error) {
	absBase, err := filepath.Abs(base)
	if err != nil {
		return false, err
	}
	absPath, err := filepath.Abs(path)
	if err != nil {
		return false, err
	}

	relPath, err := filepath.Rel(absBase, absPath)
	if err != nil {
		return false, err
	}

	// if path is below base, the relative path
	// from base to path will not start with `../`
	return !strings.HasPrefix(relPath, ".."), nil
}


forbiddenImportsFor

// forbiddenImportsFor determines all of the forbidden
// imports for a package given the import restrictions
// and returns a deduplicated list of them
func (i *ImportRestriction) forbiddenImportsFor(pkg Package) []string {
	forbiddenImportSet := map[string]struct{}{}
	imports := pkg.Imports
	if !i.ExcludeTests {
		imports = append(imports, append(pkg.TestImports, pkg.XTestImports...)...)
	}
	for _, imp := range imports {
		path := extractVendorPath(imp)
		if i.isForbidden(path) {
			forbiddenImportSet[path] = struct{}{}
		}
	}

	var forbiddenImports []string
	for imp := range forbiddenImportSet {
		forbiddenImports = append(forbiddenImports, imp)
	}
	return forbiddenImports
}


extractVendorPath

// extractVendorPath removes a vendor prefix if one exists
func extractVendorPath(path string) string {
	vendorPath := "/vendor/"
	if !strings.Contains(path, vendorPath) {
		return path
	}

	return path[strings.Index(path, vendorPath)+len(vendorPath):]
}


isForbidden

// isForbidden determines if an import is forbidden,
// which is true when the import is:
//   - of a package under the rootPackage
//   - is not of the base import path or a sub-package of it
//   - is not of an allowed path or a sub-package of one
func (i *ImportRestriction) isForbidden(imp string) bool {
	importsBelowRoot := strings.HasPrefix(imp, rootPackage)
	importsBelowBase := strings.HasPrefix(imp, i.BaseDir)
	importsAllowed := false
	for _, allowed := range i.AllowedImports {
		exactlyImportsAllowed := imp == allowed
		importsBelowAllowed := strings.HasPrefix(imp, fmt.Sprintf("%s/", allowed))
		importsAllowed = importsAllowed || (importsBelowAllowed || exactlyImportsAllowed)
	}

	return importsBelowRoot && !importsBelowBase && !importsAllowed
}


main

func main() {
	if len(os.Args) != 3 {
		log.Fatalf("Usage: %s ROOT RESTRICTIONS.yaml", os.Args[0])
	}

	rootPackage = os.Args[1]
	configFile := os.Args[2]
	importRestrictions, err := loadImportRestrictions(configFile)
	if err != nil {
		log.Fatalf("Failed to load import restrictions: %v", err)
	}

	foundForbiddenImports := false
	for _, restriction := range importRestrictions {
		log.Printf("Inspecting imports under %s...\n", restriction.BaseDir)
		packages, err := resolvePackageTree(restriction.BaseDir)
		if err != nil {
			log.Fatalf("Failed to resolve package tree: %v", err)
		} else if len(packages) == 0 {
			log.Fatalf("Found no packages under tree %s", restriction.BaseDir)
		}

		log.Printf("- validating imports for %d packages in the tree", len(packages))
		restrictionViolated := false
		for _, pkg := range packages {
			if forbidden, err := restriction.ForbiddenImportsFor(pkg); err != nil {
				log.Fatalf("-- failed to validate imports: %v", err)
			} else if len(forbidden) != 0 {
				logForbiddenPackages(pkg.ImportPath, forbidden)
				restrictionViolated = true
			}
		}
		if restrictionViolated {
			foundForbiddenImports = true
			log.Println("- FAIL")
		} else {
			log.Println("- OK")
		}
	}

	if foundForbiddenImports {
		os.Exit(1)
	}
}


loadImportRestrictions

func loadImportRestrictions(configFile string) ([]ImportRestriction, error) {
	config, err := os.ReadFile(configFile)
	if err != nil {
		return nil, fmt.Errorf("failed to load configuration from %s: %v", configFile, err)
	}

	var importRestrictions []ImportRestriction
	if err := yaml.Unmarshal(config, &importRestrictions); err != nil {
		return nil, fmt.Errorf("failed to unmarshal from %s: %v", configFile, err)
	}

	return importRestrictions, nil
}


resolvePackageTree

func resolvePackageTree(treeBase string) ([]Package, error) {
	// try resolving with $cwd
	packages, err := resolvePackageTreeInDir("", treeBase)
	if err != nil || len(packages) == 0 {
		// if that fails or finds no packages, resolve under ./vendor/$treeBase
		stagingPackages, stagingErr := resolvePackageTreeInDir(filepath.Join("./vendor", treeBase), treeBase)
		if stagingErr == nil && len(stagingPackages) > 0 {
			// if that succeeds, return
			return stagingPackages, stagingErr
		}
	}
	// otherwise, return original packages and error
	return packages, err
}


resolvePackageTreeInDir

func resolvePackageTreeInDir(dir string, treeBase string) ([]Package, error) {
	cmd := "go"
	args := []string{"list", "-json", fmt.Sprintf("%s...", treeBase)}
	c := exec.Command(cmd, args...)
	c.Dir = dir
	stdout, err := c.Output()
	if err != nil {
		var message string
		if ee, ok := err.(*exec.ExitError); ok {
			message = fmt.Sprintf("%v\n%v", ee, string(ee.Stderr))
		} else {
			message = fmt.Sprintf("%v", err)
		}
		return nil, fmt.Errorf("failed to run `%s %s`: %v", cmd, strings.Join(args, " "), message)
	}

	packages, err := decodePackages(bytes.NewReader(stdout))
	if err != nil {
		return nil, fmt.Errorf("failed to decode packages: %v", err)
	}

	return packages, nil
}


decodePackages

func decodePackages(r io.Reader) ([]Package, error) {
	// `go list -json` concatenates package definitions
	// instead of emitting a single valid JSON, so we
	// need to stream the output to decode it into the
	// data we are looking for instead of just using a
	// simple JSON decoder on stdout
	var packages []Package
	decoder := json.NewDecoder(r)
	for decoder.More() {
		var pkg Package
		if err := decoder.Decode(&pkg); err != nil {
			return nil, fmt.Errorf("invalid package: %v", err)
		}
		packages = append(packages, pkg)
	}

	return packages, nil
}


logForbiddenPackages

func logForbiddenPackages(base string, forbidden []string) {
	log.Printf("-- found forbidden imports for %s:\n", base)
	for _, forbiddenPackage := range forbidden {
		log.Printf("--- %s\n", forbiddenPackage)
	}
}


main

func main() {
	command := app.NewAPIServerCommand()
	code := cli.Run(command)
	os.Exit(code)
}


createAggregatorConfig

func createAggregatorConfig(
	kubeAPIServerConfig genericapiserver.Config,
	commandOptions *options.ServerRunOptions,
	externalInformers kubeexternalinformers.SharedInformerFactory,
	serviceResolver aggregatorapiserver.ServiceResolver,
	proxyTransport *http.Transport,
	pluginInitializers []admission.PluginInitializer,
) (*aggregatorapiserver.Config, error) {
	// make a shallow copy to let us twiddle a few things
	// most of the config actually remains the same.  We only need to mess with a couple items related to the particulars of the aggregator
	genericConfig := kubeAPIServerConfig
	genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{}
	genericConfig.RESTOptionsGetter = nil
	// prevent generic API server from installing the OpenAPI handler. Aggregator server
	// has its own customized OpenAPI handler.
	genericConfig.SkipOpenAPIInstallation = true

	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StorageVersionAPI) &&
		utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) {
		// Add StorageVersionPrecondition handler to aggregator-apiserver.
		// The handler will block write requests to built-in resources until the
		// target resources' storage versions are up-to-date.
		genericConfig.BuildHandlerChainFunc = genericapiserver.BuildHandlerChainWithStorageVersionPrecondition
	}

	// copy the etcd options so we don't mutate originals.
	// we assume that the etcd options have been completed already.  avoid messing with anything outside
	// of changes to StorageConfig as that may lead to unexpected behavior when the options are applied.
	etcdOptions := *commandOptions.Etcd
	etcdOptions.StorageConfig.Paging = utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIListChunking)
	etcdOptions.StorageConfig.Codec = aggregatorscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion, v1beta1.SchemeGroupVersion)
	etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1.SchemeGroupVersion, schema.GroupKind{Group: v1beta1.GroupName})
	etcdOptions.SkipHealthEndpoints = true // avoid double wiring of health checks
	if err := etcdOptions.ApplyTo(&genericConfig); err != nil {
		return nil, err
	}

	// override MergedResourceConfig with aggregator defaults and registry
	if err := commandOptions.APIEnablement.ApplyTo(
		&genericConfig,
		aggregatorapiserver.DefaultAPIResourceConfigSource(),
		aggregatorscheme.Scheme); err != nil {
		return nil, err
	}

	aggregatorConfig := &aggregatorapiserver.Config{
		GenericConfig: &genericapiserver.RecommendedConfig{
			Config:                genericConfig,
			SharedInformerFactory: externalInformers,
		},
		ExtraConfig: aggregatorapiserver.ExtraConfig{
			ProxyClientCertFile:       commandOptions.ProxyClientCertFile,
			ProxyClientKeyFile:        commandOptions.ProxyClientKeyFile,
			ServiceResolver:           serviceResolver,
			ProxyTransport:            proxyTransport,
			RejectForwardingRedirects: commandOptions.AggregatorRejectForwardingRedirects,
		},
	}

	// we need to clear the poststarthooks so we don't add them multiple times to all the servers (that fails)
	aggregatorConfig.GenericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{}

	return aggregatorConfig, nil
}


createAggregatorServer

func createAggregatorServer(aggregatorConfig *aggregatorapiserver.Config, delegateAPIServer genericapiserver.DelegationTarget, apiExtensionInformers apiextensionsinformers.SharedInformerFactory, crdAPIEnabled bool) (*aggregatorapiserver.APIAggregator, error) {
	aggregatorServer, err := aggregatorConfig.Complete().NewWithDelegate(delegateAPIServer)
	if err != nil {
		return nil, err
	}

	// create controllers for auto-registration
	apiRegistrationClient, err := apiregistrationclient.NewForConfig(aggregatorConfig.GenericConfig.LoopbackClientConfig)
	if err != nil {
		return nil, err
	}
	autoRegistrationController := autoregister.NewAutoRegisterController(aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), apiRegistrationClient)
	apiServices := apiServicesToRegister(delegateAPIServer, autoRegistrationController)
	crdRegistrationController := crdregistration.NewCRDRegistrationController(
		apiExtensionInformers.Apiextensions().V1().CustomResourceDefinitions(),
		autoRegistrationController)

	// Imbue all builtin group-priorities onto the aggregated discovery
	if aggregatorConfig.GenericConfig.AggregatedDiscoveryGroupManager != nil {
		for gv, entry := range apiVersionPriorities {
			aggregatorConfig.GenericConfig.AggregatedDiscoveryGroupManager.SetGroupVersionPriority(metav1.GroupVersion(gv), int(entry.group), int(entry.version))
		}
	}

	err = aggregatorServer.GenericAPIServer.AddPostStartHook("kube-apiserver-autoregistration", func(context genericapiserver.PostStartHookContext) error {
		go crdRegistrationController.Run(5, context.StopCh)
		go func() {
			// let the CRD controller process the initial set of CRDs before starting the autoregistration controller.
			// this prevents the autoregistration controller's initial sync from deleting APIServices for CRDs that still exist.
			// we only need to do this if CRDs are enabled on this server.  We can't use discovery because we are the source for discovery.
			if crdAPIEnabled {
				klog.Infof("waiting for initial CRD sync...")
				crdRegistrationController.WaitForInitialSync()
				klog.Infof("initial CRD sync complete...")
			} else {
				klog.Infof("CRD API not enabled, starting APIService registration without waiting for initial CRD sync")
			}
			autoRegistrationController.Run(5, context.StopCh)
		}()
		return nil
	})
	if err != nil {
		return nil, err
	}

	err = aggregatorServer.GenericAPIServer.AddBootSequenceHealthChecks(
		makeAPIServiceAvailableHealthCheck(
			"autoregister-completion",
			apiServices,
			aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(),
		),
	)
	if err != nil {
		return nil, err
	}

	return aggregatorServer, nil
}


makeAPIService

func makeAPIService(gv schema.GroupVersion) *v1.APIService {
	apiServicePriority, ok := apiVersionPriorities[gv]
	if !ok {
		// if we aren't found, then we shouldn't register ourselves because it could result in a CRD group version
		// being permanently stuck in the APIServices list.
		klog.Infof("Skipping APIService creation for %v", gv)
		return nil
	}
	return &v1.APIService{
		ObjectMeta: metav1.ObjectMeta{Name: gv.Version + "." + gv.Group},
		Spec: v1.APIServiceSpec{
			Group:                gv.Group,
			Version:              gv.Version,
			GroupPriorityMinimum: apiServicePriority.group,
			VersionPriority:      apiServicePriority.version,
		},
	}
}


makeAPIServiceAvailableHealthCheck

// makeAPIServiceAvailableHealthCheck returns a healthz check that returns healthy
// once all of the specified services have been observed to be available at least once.
func makeAPIServiceAvailableHealthCheck(name string, apiServices []*v1.APIService, apiServiceInformer informers.APIServiceInformer) healthz.HealthChecker {
	// Track the auto-registered API services that have not been observed to be available yet
	pendingServiceNamesLock := &sync.RWMutex{}
	pendingServiceNames := sets.NewString()
	for _, service := range apiServices {
		pendingServiceNames.Insert(service.Name)
	}

	// When an APIService in the list is seen as available, remove it from the pending list
	handleAPIServiceChange := func(service *v1.APIService) {
		pendingServiceNamesLock.Lock()
		defer pendingServiceNamesLock.Unlock()
		if !pendingServiceNames.Has(service.Name) {
			return
		}
		if v1helper.IsAPIServiceConditionTrue(service, v1.Available) {
			pendingServiceNames.Delete(service.Name)
		}
	}

	// Watch add/update events for APIServices
	apiServiceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    func(obj interface{}) { handleAPIServiceChange(obj.(*v1.APIService)) },
		UpdateFunc: func(old, new interface{}) { handleAPIServiceChange(new.(*v1.APIService)) },
	})

	// Don't return healthy until the pending list is empty
	return healthz.NamedCheck(name, func(r *http.Request) error {
		pendingServiceNamesLock.RLock()
		defer pendingServiceNamesLock.RUnlock()
		if pendingServiceNames.Len() > 0 {
			return fmt.Errorf("missing APIService: %v", pendingServiceNames.List())
		}
		return nil
	})
}


apiServicesToRegister

func apiServicesToRegister(delegateAPIServer genericapiserver.DelegationTarget, registration autoregister.AutoAPIServiceRegistration) []*v1.APIService {
	apiServices := []*v1.APIService{}

	for _, curr := range delegateAPIServer.ListedPaths() {
		if curr == "/api/v1" {
			apiService := makeAPIService(schema.GroupVersion{Group: "", Version: "v1"})
			registration.AddAPIServiceToSyncOnStart(apiService)
			apiServices = append(apiServices, apiService)
			continue
		}

		if !strings.HasPrefix(curr, "/apis/") {
			continue
		}
		// this comes back in a list that looks like /apis/rbac.authorization.k8s.io/v1alpha1
		tokens := strings.Split(curr, "/")
		if len(tokens) != 4 {
			continue
		}

		apiService := makeAPIService(schema.GroupVersion{Group: tokens[2], Version: tokens[3]})
		if apiService == nil {
			continue
		}
		registration.AddAPIServiceToSyncOnStart(apiService)
		apiServices = append(apiServices, apiService)
	}

	return apiServices
}