Restart LXD on MicroCloud start to refresh symlinks

microcloud/cmd/microcloud/main_init.go

@@ -51,6 +51,18 @@ func (c *cmdInit) Run(cmd *cobra.Command, args []string) error {
 		return cmd.Help()
 	}
 
+	// Initially restart LXD so that the correct MicroCloud service related state is set by the LXD snap.
+	fmt.Println("Waiting for LXD to start...")
+	lxdService, err := service.NewLXDService(context.Background(), "", "", c.common.FlagMicroCloudDir)
+	if err != nil {
+		return err
+	}
+
+	err = lxdService.Restart(30)
+	if err != nil {
+		return err
+	}
+
 	addr, subnet, err := askAddress(c.flagAutoSetup, c.flagAddress)
 	if err != nil {
 		return err

microcloud/service/lxd.go

@@ -9,11 +9,13 @@ import (
 	"net/url"
 	"os"
 	"strings"
+	"time"
 
 	lxd "github.com/canonical/lxd/client"
 	"github.com/canonical/lxd/lxd/util"
 	"github.com/canonical/lxd/shared"
 	"github.com/canonical/lxd/shared/api"
+	"github.com/canonical/lxd/shared/logger"
 	"github.com/canonical/microcluster/microcluster"
 
 	"github.com/canonical/microcloud/microcloud/api/types"
@@ -150,6 +152,11 @@ func (s LXDService) Bootstrap() error {
 
 // Join joins a cluster with the given token.
 func (s LXDService) Join(joinConfig JoinConfig) error {
+	err := s.Restart(30)
+	if err != nil {
+		return err
+	}
+
 	config, err := s.configFromToken(joinConfig.Token)
 	if err != nil {
 		return err
@@ -680,6 +687,102 @@ func (s *LXDService) SetConfig(target string, secret string, config map[string]s
 	return c.UpdateServer(newServer, "")
 }
 
+// isInitialized checks if LXD is initialized by fetching the storage pools.
+// If none exist, that means LXD has not yet been set up.
+func (s *LXDService) isInitialized(c lxd.InstanceServer) (bool, error) {
+	pools, err := c.GetStoragePoolNames()
+	if err != nil {
+		return false, err
+	}
+
+	return len(pools) != 0, nil
+}
+
+// Restart requests LXD to shutdown, then waits until it is ready.
+func (s *LXDService) Restart(timeoutSeconds int) error {
+	c, err := s.client("")
+	if err != nil {
+		return err
+	}
+
+	isInit, err := s.isInitialized(c)
+	if err != nil {
+		return fmt.Errorf("Failed to check LXD initialization: %w", err)
+	}
+
+	if isInit {
+		return fmt.Errorf("LXD has already been initialized")
+	}
+
+	_, _, err = c.RawQuery("PUT", "/internal/shutdown", nil, "")
+	if err != nil {
+		return fmt.Errorf("Failed to send shutdown request to LXD: %w", err)
+	}
+
+	err = s.waitReady(c, timeoutSeconds)
+	if err != nil {
+		return err
+	}
+
+	// A sleep might be necessary here on slower machines?
+	_, _, err = c.GetServer()
+	if err != nil {
+		return fmt.Errorf("Failed to initialize LXD server: %w", err)
+	}
+
+	return nil
+}
+
+// waitReady repeatedly (500ms intervals) asks LXD if it is ready, up to the given timeout.
+func (s *LXDService) waitReady(c lxd.InstanceServer, timeoutSeconds int) error {
+	finger := make(chan error, 1)
+	var errLast error
+	go func() {
+		for i := 0; ; i++ {
+			// Start logging only after the 10'th attempt (about 5
+			// seconds). Then after the 30'th attempt (about 15
+			// seconds), log only only one attempt every 10
+			// attempts (about 5 seconds), to avoid being too
+			// verbose.
+			doLog := false
+			if i > 10 {
+				doLog = i < 30 || ((i % 10) == 0)
+			}
+
+			if doLog {
+				logger.Debugf("Checking if LXD daemon is ready (attempt %d)", i)
+			}
+
+			_, _, err := c.RawQuery("GET", "/internal/ready", nil, "")
+			if err != nil {
+				errLast = err
+				if doLog {
+					logger.Warnf("Failed to check if LXD daemon is ready (attempt %d): %v", i, err)
+				}
+
+				time.Sleep(500 * time.Millisecond)
+				continue
+			}
+
+			finger <- nil
+			return
+		}
+	}()
+
+	if timeoutSeconds > 0 {
+		select {
+		case <-finger:
+			break
+		case <-time.After(time.Second * time.Duration(timeoutSeconds)):
+			return fmt.Errorf("LXD is still not running after %ds timeout (%v)", timeoutSeconds, errLast)
+		}
+	} else {
+		<-finger
+	}
+
+	return nil
+}
+
 // defaultGatewaySubnetV4 returns subnet of default gateway interface.
 func defaultGatewaySubnetV4() (*net.IPNet, string, error) {
 	file, err := os.Open("/proc/net/route")

microcloud/service/service_handler.go

@@ -8,8 +8,8 @@ import (
 	"path/filepath"
 	"sync"
 
-	lxd "github.com/canonical/lxd/client"
 	"github.com/canonical/lxd/shared"
+	"github.com/canonical/lxd/shared/logger"
 	"github.com/canonical/microcluster/state"
 	"github.com/hashicorp/mdns"
 
@@ -84,10 +84,9 @@ func (s *Handler) Start(state *state.State) error {
 		return nil
 	}
 
-	// Attempt to wake up LXD so it can generate certificates already.
-	d, err := lxd.ConnectLXDUnix("/var/snap/lxd/common/lxd/unix.socket", nil)
-	if err == nil {
-		_, _, _ = d.GetServer()
+	err := s.Services[types.LXD].(*LXDService).Restart(30)
+	if err != nil {
+		logger.Error("Failed to restart LXD", logger.Ctx{"error": err})
 	}
 
 	s.AuthSecret, err = shared.RandomCryptoString()