From 55c2a8fe66d622577ffbbc2598ecc8cff38b5555 Mon Sep 17 00:00:00 2001 From: Max Asnaashari Date: Mon, 24 Jul 2023 16:12:38 +0000 Subject: [PATCH 1/3] microcloud/service/lxd: Add Restart helper to LXD service Signed-off-by: Max Asnaashari --- microcloud/service/lxd.go | 78 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/microcloud/service/lxd.go b/microcloud/service/lxd.go index eabd39487..eda32289f 100644 --- a/microcloud/service/lxd.go +++ b/microcloud/service/lxd.go @@ -9,11 +9,13 @@ import ( "net/url" "os" "strings" + "time" lxd "github.com/canonical/lxd/client" "github.com/canonical/lxd/lxd/util" "github.com/canonical/lxd/shared" "github.com/canonical/lxd/shared/api" + "github.com/canonical/lxd/shared/logger" "github.com/canonical/microcluster/microcluster" "github.com/canonical/microcloud/microcloud/api/types" @@ -680,6 +682,82 @@ func (s *LXDService) SetConfig(target string, secret string, config map[string]s return c.UpdateServer(newServer, "") } +// Restart requests LXD to shutdown, then waits until it is ready. +func (s *LXDService) Restart(timeoutSeconds int) error { + c, err := s.client("") + if err != nil { + return err + } + + _, _, err = c.RawQuery("PUT", "/internal/shutdown", nil, "") + if err != nil { + return fmt.Errorf("Failed to send shutdown request to LXD: %w", err) + } + + err = s.waitReady(c, timeoutSeconds) + if err != nil { + return err + } + + // A sleep might be necessary here on slower machines? + _, _, err = c.GetServer() + if err != nil { + return fmt.Errorf("Failed to initialize LXD server: %w", err) + } + + return nil +} + +// waitReady repeatedly (500ms intervals) asks LXD if it is ready, up to the given timeout. +func (s *LXDService) waitReady(c lxd.InstanceServer, timeoutSeconds int) error { + finger := make(chan error, 1) + var errLast error + go func() { + for i := 0; ; i++ { + // Start logging only after the 10'th attempt (about 5 + // seconds). Then after the 30'th attempt (about 15 + // seconds), log only only one attempt every 10 + // attempts (about 5 seconds), to avoid being too + // verbose. + doLog := false + if i > 10 { + doLog = i < 30 || ((i % 10) == 0) + } + + if doLog { + logger.Debugf("Checking if LXD daemon is ready (attempt %d)", i) + } + + _, _, err := c.RawQuery("GET", "/internal/ready", nil, "") + if err != nil { + errLast = err + if doLog { + logger.Warnf("Failed to check if LXD daemon is ready (attempt %d): %v", i, err) + } + + time.Sleep(500 * time.Millisecond) + continue + } + + finger <- nil + return + } + }() + + if timeoutSeconds > 0 { + select { + case <-finger: + break + case <-time.After(time.Second * time.Duration(timeoutSeconds)): + return fmt.Errorf("LXD is still not running after %ds timeout (%v)", timeoutSeconds, errLast) + } + } else { + <-finger + } + + return nil +} + // defaultGatewaySubnetV4 returns subnet of default gateway interface. func defaultGatewaySubnetV4() (*net.IPNet, string, error) { file, err := os.Open("/proc/net/route") From b9681a8b761e769ffc10dbf3f430388d9f42ac61 Mon Sep 17 00:00:00 2001 From: Max Asnaashari Date: Mon, 24 Jul 2023 16:12:46 +0000 Subject: [PATCH 2/3] microcloud/service/service/handler: Restart LXD before init Sets LXD to restart at three points: 1) When MicroCloud first starts (to avoid a potential lag on all peers when fetching resources as LXD sets up for the first time) 2) When the user invokes `microcloud init`, to ensure LXD is set up properly with the correct symlinks for microcloud 3) When a MicroCloud daemon receives a join request for LXD, to ensure the above for all joining nodes. Signed-off-by: Max Asnaashari --- microcloud/cmd/microcloud/main_init.go | 12 ++++++++++++ microcloud/service/lxd.go | 5 +++++ microcloud/service/service_handler.go | 9 ++++----- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/microcloud/cmd/microcloud/main_init.go b/microcloud/cmd/microcloud/main_init.go index 8ac15fc44..253f2df05 100644 --- a/microcloud/cmd/microcloud/main_init.go +++ b/microcloud/cmd/microcloud/main_init.go @@ -51,6 +51,18 @@ func (c *cmdInit) Run(cmd *cobra.Command, args []string) error { return cmd.Help() } + // Initially restart LXD so that the correct MicroCloud service related state is set by the LXD snap. + fmt.Println("Waiting for LXD to start...") + lxdService, err := service.NewLXDService(context.Background(), "", "", c.common.FlagMicroCloudDir) + if err != nil { + return err + } + + err = lxdService.Restart(30) + if err != nil { + return err + } + addr, subnet, err := askAddress(c.flagAutoSetup, c.flagAddress) if err != nil { return err diff --git a/microcloud/service/lxd.go b/microcloud/service/lxd.go index eda32289f..202849eb8 100644 --- a/microcloud/service/lxd.go +++ b/microcloud/service/lxd.go @@ -152,6 +152,11 @@ func (s LXDService) Bootstrap() error { // Join joins a cluster with the given token. func (s LXDService) Join(joinConfig JoinConfig) error { + err := s.Restart(30) + if err != nil { + return err + } + config, err := s.configFromToken(joinConfig.Token) if err != nil { return err diff --git a/microcloud/service/service_handler.go b/microcloud/service/service_handler.go index ed8c35bd6..d04ad85d6 100644 --- a/microcloud/service/service_handler.go +++ b/microcloud/service/service_handler.go @@ -8,8 +8,8 @@ import ( "path/filepath" "sync" - lxd "github.com/canonical/lxd/client" "github.com/canonical/lxd/shared" + "github.com/canonical/lxd/shared/logger" "github.com/canonical/microcluster/state" "github.com/hashicorp/mdns" @@ -84,10 +84,9 @@ func (s *Handler) Start(state *state.State) error { return nil } - // Attempt to wake up LXD so it can generate certificates already. - d, err := lxd.ConnectLXDUnix("/var/snap/lxd/common/lxd/unix.socket", nil) - if err == nil { - _, _, _ = d.GetServer() + err := s.Services[types.LXD].(*LXDService).Restart(30) + if err != nil { + logger.Error("Failed to restart LXD", logger.Ctx{"error": err}) } s.AuthSecret, err = shared.RandomCryptoString() From 80f4f03e94aabee8b8bedf420994299effe0860b Mon Sep 17 00:00:00 2001 From: Max Asnaashari Date: Mon, 24 Jul 2023 16:25:40 +0000 Subject: [PATCH 3/3] microcloud/service/lxd: Check if LXD is initialized before restarting Signed-off-by: Max Asnaashari --- microcloud/service/lxd.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/microcloud/service/lxd.go b/microcloud/service/lxd.go index 202849eb8..58f461412 100644 --- a/microcloud/service/lxd.go +++ b/microcloud/service/lxd.go @@ -687,6 +687,17 @@ func (s *LXDService) SetConfig(target string, secret string, config map[string]s return c.UpdateServer(newServer, "") } +// isInitialized checks if LXD is initialized by fetching the storage pools. +// If none exist, that means LXD has not yet been set up. +func (s *LXDService) isInitialized(c lxd.InstanceServer) (bool, error) { + pools, err := c.GetStoragePoolNames() + if err != nil { + return false, err + } + + return len(pools) != 0, nil +} + // Restart requests LXD to shutdown, then waits until it is ready. func (s *LXDService) Restart(timeoutSeconds int) error { c, err := s.client("") @@ -694,6 +705,15 @@ func (s *LXDService) Restart(timeoutSeconds int) error { return err } + isInit, err := s.isInitialized(c) + if err != nil { + return fmt.Errorf("Failed to check LXD initialization: %w", err) + } + + if isInit { + return fmt.Errorf("LXD has already been initialized") + } + _, _, err = c.RawQuery("PUT", "/internal/shutdown", nil, "") if err != nil { return fmt.Errorf("Failed to send shutdown request to LXD: %w", err)