feat: specify an ovn underlay network

api/services.go

@@ -125,7 +125,7 @@ func servicesPut(state *state.State, r *http.Request) response.Response {
 	services := make([]types.ServiceType, len(req.Tokens))
 	for i, cfg := range req.Tokens {
 		services[i] = types.ServiceType(cfg.Service)
-		joinConfigs[cfg.Service] = service.JoinConfig{Token: cfg.JoinToken, LXDConfig: req.LXDConfig, CephConfig: req.CephConfig}
+		joinConfigs[cfg.Service] = service.JoinConfig{Token: cfg.JoinToken, LXDConfig: req.LXDConfig, CephConfig: req.CephConfig, OVNConfig: req.OVNConfig}
 	}
 
 	// Default to the first iface if none specified.

api/types/services.go

@@ -29,6 +29,7 @@ type ServicesPut struct {
 
 	LXDConfig  []api.ClusterMemberConfigKey `json:"lxd_config" yaml:"lxd_config"`
 	CephConfig []types.DisksPost            `json:"ceph_config" yaml:"ceph_config"`
+	OVNConfig  map[string]string            `json:"ovn_config" yaml:"ovn_config"`
 }
 
 // ServiceToken represents a join token for a service join request.

cmd/microcloud/ask.go

@@ -1045,6 +1045,104 @@ func (c *CmdControl) askNetwork(sh *service.Handler, systems map[string]InitSyst
 		systems[sh.Name] = bootstrapSystem
 	}
 
+	// Underlay network.
+	underlayNetworks, err := sh.Services[types.LXD].(*service.LXDService).GetUnderlayInterfaces(context.Background(), bootstrap, infos)
+	if err != nil {
+		return err
+	}
+
+	if len(underlayNetworks) != len(systems) {
+		fmt.Println("Not enough interfaces available to create an underlay network, skipping")
+		return nil
+	}
+
+	canOVNUnderlay := len(underlayNetworks) > 0
+	for _, nets := range underlayNetworks {
+		if len(nets) == 0 {
+			canOVNUnderlay = false
+			break
+		}
+	}
+
+	if !canOVNUnderlay {
+		fmt.Println("No dedicated interfaces for OVN underlay detected, skipping the OVN underlay network setup")
+		return nil
+	}
+
+	// // Check if the current MicroOVN deployment supports custom encapsulation IP for the Geneve tunnel.
+	// hasFeatureCustomIPEncapsulationFeature, err := sh.Services[types.MicroOVN].(*service.OVNService).SupportsFeature(context.Background(), "custom_encapsulation_ip")
+	// if err != nil {
+	// 	return err
+	// }
+
+	// if !hasFeatureCustomIPEncapsulationFeature {
+	// 	fmt.Println("Underlay network setup for MicroOVN was not applied because your MicroOVN memmbers do not support custom encapsulation IP for the Geneve tunnel, skipping")
+	// 	return nil
+	// }
+
+	wantsDedicatedUnderlay, err := c.asker.AskBool("Configure dedicated underlay networking? (yes/no) [default=no]: ", "no")
+	if err != nil {
+		return err
+	}
+
+	if !wantsDedicatedUnderlay {
+		return nil
+	}
+
+	header = []string{"LOCATION", "IFACE", "TYPE", "IP ADDRESS (CIDR)"}
+	fmt.Println("Select exactly one network interface from each cluster member:")
+	data = [][]string{}
+	for peer, nets := range underlayNetworks {
+		for _, net := range nets {
+			for _, addr := range net.Addresses {
+				data = append(data, []string{peer, net.Network.Name, net.Network.Type, addr})
+			}
+		}
+	}
+
+	table = NewSelectableTable(header, data)
+	selected = map[string]string{}
+	c.askRetry("Retry selecting underlay network interfaces?", autoSetup, func() error {
+		err = table.Render(table.rows)
+		if err != nil {
+			return err
+		}
+
+		answers, err := table.GetSelections()
+		if err != nil {
+			return err
+		}
+
+		selected = map[string]string{}
+		for _, answer := range answers {
+			target := table.SelectionValue(answer, "LOCATION")
+			ipAddr := table.SelectionValue(answer, "IP ADDRESS (CIDR)")
+
+			if selected[target] != "" {
+				return fmt.Errorf("Failed to configure OVN underlay traffic: Selected more than one interface for target %q", target)
+			}
+
+			selected[target] = ipAddr
+		}
+
+		if len(selected) != len(underlayNetworks) {
+			return fmt.Errorf("Failed to configure OVN underlay traffic: Some peers don't have a selected interface")
+		}
+
+		return nil
+	})
+
+	for peer, ipAddr := range selected {
+		system := systems[peer]
+		ip, _, err := net.ParseCIDR(ipAddr)
+		if err != nil {
+			return err
+		}
+
+		system.OVNGeneveAddr = ip.String()
+		systems[peer] = system
+	}
+
 	return nil
 }
 

cmd/microcloud/main_init.go

@@ -45,6 +45,9 @@ type InitSystem struct {
 	TargetStoragePools []lxdAPI.StoragePoolsPost
 	// Networks is the cluster-wide network configuration.
 	Networks []lxdAPI.NetworksPost
+	// OVNGeneveAddr represents an IP address to use for the OVN (if OVN is supported) Geneve tunnel on this system.
+	// If left empty, the system will choose to route the Geneve traffic through the management network.
+	OVNGeneveAddr string
 	// StoragePools is the cluster-wide storage pool configuration.
 	StoragePools []lxdAPI.StoragePoolsPost
 	// StorageVolumes is the cluster-wide storage volume configuration.
@@ -95,7 +98,7 @@ func (c *cmdInit) Run(cmd *cobra.Command, args []string) error {
 func (c *cmdInit) RunInteractive(cmd *cobra.Command, args []string) error {
 	// Initially restart LXD so that the correct MicroCloud service related state is set by the LXD snap.
 	fmt.Println("Waiting for LXD to start...")
-	lxdService, err := service.NewLXDService("", "", c.common.FlagMicroCloudDir)
+	lxdService, err := service.NewLXDService("", "", c.common.FlagMicroCloudDir, nil)
 	if err != nil {
 		return err
 	}
@@ -407,6 +410,12 @@ func AddPeers(sh *service.Handler, systems map[string]InitSystem) error {
 			LXDConfig:  info.JoinConfig,
 			CephConfig: info.MicroCephDisks,
 		}
+
+		if info.OVNGeneveAddr != "" {
+			p := joinConfig[peer]
+			p.OVNConfig = map[string]string{fmt.Sprintf("%s.ovn-encap-ip", info.ServerInfo.Name): info.OVNGeneveAddr}
+			joinConfig[peer] = p
+		}
 	}
 
 	_, bootstrap := systems[sh.Name]
@@ -720,6 +729,20 @@ func setupCluster(s *service.Handler, systems map[string]InitSystem) error {
 				}
 			}
 
+			if s.Type() == types.MicroOVN {
+				microOvnBootstrapConf := make(map[string]string)
+				for _, system := range systems {
+					// If the system needs to be bootstrapped with a custom ovn encapsulation ip for geneve, add it to the init config.
+					if system.OVNGeneveAddr != "" {
+						microOvnBootstrapConf[fmt.Sprintf("%s.ovn-encap-ip", system.ServerInfo.Name)] = system.OVNGeneveAddr
+					}
+				}
+
+				if len(microOvnBootstrapConf) > 0 {
+					s.SetConfig(microOvnBootstrapConf)
+				}
+			}
+
 			// set a 2 minute timeout to bootstrap a service in case the node is slow.
 			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
 			defer cancel()

cmd/microcloud/main_init_preseed.go

@@ -38,6 +38,7 @@ type Preseed struct {
 type System struct {
 	Name            string      `yaml:"name"`
 	UplinkInterface string      `yaml:"ovn_uplink_interface"`
+	UnderlayIP      string      `yaml:"underlay_ip"`
 	Storage         InitStorage `yaml:"storage"`
 }
 
@@ -180,6 +181,7 @@ func (c *CmdControl) RunPreseed(cmd *cobra.Command, init bool) error {
 // validate validates the unmarshaled preseed input.
 func (p *Preseed) validate(name string, bootstrap bool) error {
 	uplinkCount := 0
+	underlayCount := 0
 	directCephCount := 0
 	directLocalCount := 0
 	localInit := false
@@ -205,6 +207,15 @@ func (p *Preseed) validate(name string, bootstrap bool) error {
 			uplinkCount++
 		}
 
+		if system.UnderlayIP != "" {
+			_, _, err := net.ParseCIDR(system.UnderlayIP)
+			if err != nil {
+				return fmt.Errorf("Invalid underlay IP: %w", err)
+			}
+
+			underlayCount++
+		}
+
 		if len(system.Storage.Ceph) > 0 {
 			directCephCount++
 		}
@@ -234,6 +245,11 @@ func (p *Preseed) validate(name string, bootstrap bool) error {
 		return fmt.Errorf("Some systems are missing an uplink interface")
 	}
 
+	containsUnderlay := underlayCount > 0
+	if containsUnderlay && underlayCount < len(p.Systems) {
+		return fmt.Errorf("Some systems are missing an underlay interface")
+	}
+
 	containsCephStorage = directCephCount > 0
 	if containsCephStorage && directCephCount < 3 && len(p.Storage.Ceph) == 0 && bootstrap {
 		return fmt.Errorf("At least 3 systems must specify ceph storage disks")
@@ -451,10 +467,15 @@ func (p *Preseed) Parse(s *service.Handler, bootstrap bool) (map[string]InitSyst
 
 	lxd := s.Services[types.LXD].(*service.LXDService)
 	ifaceByPeer := map[string]string{}
+	ovnUnderlayNeeded := false
 	for _, cfg := range p.Systems {
 		if cfg.UplinkInterface != "" {
 			ifaceByPeer[cfg.Name] = cfg.UplinkInterface
 		}
+
+		if cfg.UnderlayIP != "" {
+			ovnUnderlayNeeded = true
+		}
 	}
 
 	// If we have specified any part of OVN config, implicitly assume we want to set it up.
@@ -479,10 +500,58 @@ func (p *Preseed) Parse(s *service.Handler, bootstrap bool) (map[string]InitSyst
 				ifaceByPeer[peer] = nets[0].Name
 			}
 		}
-	}
 
-	if usingOVN && bootstrap && len(ifaceByPeer) < 3 {
-		return nil, fmt.Errorf("Failed to find at least 3 interfaces on 3 machines for OVN configuration")
+		if bootstrap && len(ifaceByPeer) < 3 {
+			return nil, fmt.Errorf("Failed to find at least 3 interfaces on 3 machines for OVN configuration")
+		}
+
+		// Check the preseed underlay network configuration against the available ips.
+		if ovnUnderlayNeeded {
+			underlayNetworks, err := lxd.GetUnderlayInterfaces(context.Background(), bootstrap, infos)
+			if err != nil {
+				return nil, err
+			}
+
+			if len(underlayNetworks) != len(p.Systems) {
+				return nil, fmt.Errorf("Failed to find all underlay IPs on the network")
+			}
+
+			underlays := make(map[string]string, len(p.Systems))
+			for _, sys := range p.Systems {
+				underlays[sys.Name] = sys.UnderlayIP
+			}
+
+			underlayCount := 0
+			for peer, networks := range underlayNetworks {
+				for _, net := range networks {
+					if len(net.Addresses) != 0 {
+						for _, cidrAddr := range net.Addresses {
+							if underlays[peer] == cidrAddr {
+								underlayCount = underlayCount + 1
+								goto out
+							}
+						}
+					}
+				}
+
+			out:
+			}
+
+			if underlayCount != len(p.Systems) {
+				return nil, fmt.Errorf("Failed to find all underlay IPs on the network")
+			}
+
+			// Apply the underlay IPs to the systems.
+			for peer, system := range systems {
+				ip, _, err := net.ParseCIDR(underlays[peer])
+				if err != nil {
+					return nil, fmt.Errorf("Failed to parse underlay IP: %w", err)
+				}
+
+				system.OVNGeneveAddr = ip.String()
+				systems[peer] = system
+			}
+		}
 	}
 
 	for peer, iface := range ifaceByPeer {

cmd/microcloudd/main.go

@@ -101,7 +101,7 @@ func (c *cmdDaemon) Run(cmd *cobra.Command, args []string) error {
 
 					newService, err := service.NewHandler(name, addr, c.flagMicroCloudDir, false, false, serviceName)
 					if err != nil {
-						logger.Error("Failed to create servie handler for service", logger.Ctx{"service": serviceName, "error": err})
+						logger.Error("Failed to create service handler for service", logger.Ctx{"service": serviceName, "error": err})
 						break
 					}
 

doc/explanation/microcloud.md

@@ -61,6 +61,19 @@ MicroCloud will still be usable, but you will see some limitations:
   As a result of this, network forwarding works at a basic level only, and external addresses must be forwarded to a specific cluster member and don't fail over.
 - There is no support for hardware acceleration, load balancers, or ACL functionality within the local network.
 
+(microcloud-networking-underlay)=
+
+Network interface to connect to an OVN underlay network
+:  While setting up the distributed networking with OVN, you can also decide to set up a dedicated underlay network for the OVN traffic.
+   A dedicated underlay network for OVN serves as the physical infrastructure over which the virtual networking (overlay network) is constructed. It can be beneficial to set up a dedicated underlay network for the following reasons:
+
+- A dedicated underlay network isolates the overlay network traffic from other types of traffic (such as management traffic).
+- By dedicating a physical network solely for the purpose of the virtual network overlay, network congestion can be reduced.
+- A dedicated underlay network can be tuned to provide predictable latency and bandwidth characteristics, which are crucial for the performance of latency-sensitive applications running over the OVN overlay.
+- A dedicated underlay can be designed to scale independently of other networks, which allows for more efficient scaling of the overlay network as the demand for virtual network resources increases.
+
+See {ref}`howto-ovn-underlay` for how to set up a dedicated OVN underlay network.
+
 ### Dedicated internal network for Ceph
 You can set up a dedicated network for Ceph to separate the internal Ceph traffic from the rest of the MicroCloud cluster traffic.
 

doc/how-to/index.md

@@ -10,6 +10,7 @@ Install MicroCloud </how-to/install>
 Manage the snaps </how-to/snaps>
 Initialise MicroCloud </how-to/initialise>
 Configure Ceph networking </how-to/ceph_networking>
+Configure OVN underlay </how-to/ovn_underlay>
 Add a machine </how-to/add_machine>
 Get support </how-to/support>
 Contribute to MicroCloud </how-to/contribute>

doc/how-to/initialise.md

@@ -90,6 +90,13 @@ Complete the following steps to initialise MicroCloud:
       You must select one network interface per machine.
    1. If you want to use IPv4, specify the IPv4 gateway on the uplink network (in CIDR notation) and the first and last IPv4 address in the range that you want to use with LXD.
    1. If you want to use IPv6, specify the IPv6 gateway on the uplink network (in CIDR notation).
+   1. If you chose to set up distributed networking, you can choose to setup an underlay network for the distributed networking:
+
+      If you choose ``yes``, configure the underlay network:
+
+      1. Select the network interfaces that you want to use (see {ref}`microcloud-networking-underlay`).
+
+         You must select one network interface with an IP address per machine.
 1. MicroCloud now starts to bootstrap the cluster.
    Monitor the output to see whether all steps complete successfully.
    See {ref}`bootstrapping-process` for more information.