Reconcile etcd endpoint's peer URL for single-member clusters

Previously, k0s only ever used the etcd peer address during local etcd
member initialization, either during etcd cluster bootstrap, or when
joining a new member into an existing cluster.

During k0s startup, allow for the peer address to be changed
automatically for single-node clusters. This addresses a design issue
in k0s restore where the etcd cluster is restored using either the
backed-up etcd peer address or the auto-detected node's peer address if
the backed-up node didn't specify one. Without this change, there was
no way of restoring a node while also changing the node IP. This is now
possible by simply modifying the local k0s config before starting the
restored node.

Note that this is _only_ done for single-member etcd clusters. For
multi-member etcd clusters, the join API performs a basic reachability
check before adding a new member, to prevent cluster outages caused by
bad configuration values. While this might be doable here as well, it
would require significantly more effort, like trying to communicate with
the join API, potentially adding a new endpoint. Postpone that decision
for now.

Signed-off-by: Tom Wieczorek <twieczorek@mirantis.com>

docs/backup.md

@@ -53,6 +53,17 @@ k0s restore /tmp/k0s_backup_2021-04-26T19_51_57_000Z.tar.gz
 The command would fail if the data directory for the current controller has overlapping data with the backup archive content.
 
 The command would use the archived `k0s.yaml` as the cluster configuration description.
+It is written to the current working directory as `k0s_<archive timestamp>.yaml`
+(use `--config-out` to change this).
+
+If the backup is restored onto a node with a different IP address than the one
+it was taken on, and the archived configuration pins the etcd peer address via
+`spec.storage.etcd.peerAddress`, adjust it before starting the controller:
+either set it to the new node's address or remove it to have it detected
+automatically. Then start the controller with `--config` pointing at the
+adjusted file. k0s will update the peer URL of the restored etcd member
+accordingly on startup. This only works as long as the restored controller is
+the sole etcd member, i.e. before any additional controllers are joined.
 
 In case if your cluster is HA, after restoring single controller node, join the rest of the controller nodes to the cluster.
 E.g. steps for N nodes cluster would be:

pkg/component/controller/etcd.go

@@ -145,7 +145,7 @@ func (e *Etcd) syncEtcdConfig(ctx context.Context, etcdRequest v1beta1.EtcdReque
 }
 
 // Run runs etcd if external cluster is not configured
-func (e *Etcd) Start(ctx context.Context) error {
+func (e *Etcd) Start(ctx context.Context) (err error) {
 	etcdCaCert := filepath.Join(e.K0sVars.EtcdCertDir, "ca.crt")
 	etcdCaCertKey := filepath.Join(e.K0sVars.EtcdCertDir, "ca.key")
 	etcdServerCert := filepath.Join(e.K0sVars.EtcdCertDir, "server.crt")
@@ -243,7 +243,85 @@ func (e *Etcd) Start(ctx context.Context) error {
 		KeepEnvPrefix: true,
 	}
 
-	return e.supervisor.Supervise(ctx)
+	if err := e.supervisor.Supervise(ctx); err != nil {
+		return err
+	}
+	defer func() {
+		if err != nil {
+			err = errors.Join(err, e.supervisor.Stop())
+		}
+	}()
+
+	for nextWarn := time.Now().Add(10 * time.Second); ; {
+		select {
+		case <-time.After(1 * time.Second):
+		case <-ctx.Done():
+			return fmt.Errorf("while checking peer URL: %w", context.Cause(ctx))
+		}
+
+		ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+		err := e.fixupPeerURL(ctx)
+		cancel()
+		if err == nil {
+			return nil
+		}
+
+		// Failures are expected while etcd is still starting up. Log them
+		// quietly at first, but raise the log level now and then if this takes
+		// longer than usual, so that a stalled startup (e.g. an etcd cluster
+		// waiting for quorum) doesn't go unnoticed in non-debug logs.
+		level := logrus.DebugLevel
+		if now := time.Now(); !now.Before(nextWarn) {
+			level, nextWarn = logrus.WarnLevel, now.Add(10*time.Second)
+		}
+		logrus.WithError(err).Log(level, "Failed to check peer URL, retrying")
+	}
+}
+
+func (e *Etcd) fixupPeerURL(ctx context.Context) (err error) {
+	c, err := etcd.NewClient(e.K0sVars.CertRootDir, e.K0sVars.EtcdCertDir, e.Config)
+	if err != nil {
+		return err
+	}
+	defer func() { err = errors.Join(err, c.Close()) }()
+
+	status, err := c.Status(ctx)
+	if err != nil {
+		return err
+	}
+
+	memberList, err := c.ListMembers(ctx)
+	if err != nil {
+		return err
+	}
+
+	var clusterPeerURL string
+	for _, member := range memberList {
+		if member.ID == status.ID {
+			clusterPeerURL = member.PeerURL
+			break
+		}
+	}
+	if clusterPeerURL == "" {
+		return fmt.Errorf("local endpoint %x not in member list", status.ID)
+	}
+
+	peerURL := e.Config.GetPeerURL()
+	if clusterPeerURL == peerURL {
+		return nil
+	}
+
+	if len(memberList) != 1 {
+		logrus.Warnf("Unexpected peer URL for local etcd endpoint %x: %s", status.ID, clusterPeerURL)
+		return nil
+	}
+
+	if err := c.SetPeerURL(ctx, status.ID, peerURL); err != nil {
+		return fmt.Errorf("failed to change peer URL for local endpoint %x from %s to %s: %w", status.ID, clusterPeerURL, peerURL, err)
+	}
+	logrus.Infof("Changed peer URL for local etcd endpoint %x from %s to %s", status.ID, clusterPeerURL, peerURL)
+
+	return nil
 }
 
 // Stop stops etcd

pkg/etcd/client.go

@@ -207,6 +207,11 @@ func (c *Client) PromoteMember(ctx context.Context, memberID uint64) error {
 	return err
 }
 
+func (c *Client) SetPeerURL(ctx context.Context, memberID uint64, peerURL string) error {
+	_, err := c.client.MemberUpdate(ctx, memberID, []string{peerURL})
+	return err
+}
+
 // Close closes the etcd client
 func (c *Client) Close() error {
 	return c.client.Close()