Device: Don't remove an existing target directory when unmounting a disk device if the original dir hasn't been created by LXD

doc/api-extensions.md

@@ -2419,3 +2419,7 @@ Adds the ability to use an unspecified IPv4 (`0.0.0.0`) or IPv6 (`::`) address i
 If an unspecified IP address is used, supported drivers will allocate an available listen address automatically.
 Allocation of external IP addresses is currently supported by the OVN network driver.
 The OVN driver will allocate IP addresses from the subnets specified in the uplink network's `ipv4.routes` and `ipv6.routes` configuration options.
+
+## `disk_state_created`
+
+This API extension provides the ability to check if a target directory was created within the instance file system at mount time. If a host directory is mounted to an existing container directory (e.g., `/opt`), the target directory won't be removed upon unmounting. However, if LXD creates a target directory during the mount, like `/new_dir`, it will be deleted when the device is unmounted.

doc/config_options.txt

@@ -2389,6 +2389,14 @@ Specify either a cron expression (`<minute> <hour> <dom> <month> <dow>`), a comm
 
 <!-- config group instance-snapshots end -->
 <!-- config group instance-volatile start -->
+```{config:option} volatile.<disk_dev_name>.last_state.created instance-volatile
+:shortdesc: "Path of the directory created from mounting a `disk` device"
+:type: "string"
+If mounting a host directory through a `disk` device creates the target directory, this option contains the path of this newly created target directory.
+For example, if the target directory is `/opt` and `/opt` already exists in the instance, this key is not set.
+However, if the target directory is `/opt/foo` and `/opt/foo` doesn't exist in the instance, this key is set to `/opt/foo`.
+```
+
 ```{config:option} volatile.<name>.apply_quota instance-volatile
 :shortdesc: "Disk quota"
 :type: "string"

doc/rest-api.yaml

@@ -1,4 +1,29 @@
 definitions:
+    AgentDeviceRemove:
+        properties:
+            config:
+                additionalProperties:
+                    type: string
+                description: Device configuration map
+                type: object
+                x-go-name: Config
+            name:
+                description: Device name
+                type: string
+                x-go-name: Name
+            type:
+                description: Type of device ('disk', 'nic', etc.)
+                type: string
+                x-go-name: Type
+            volatile:
+                additionalProperties:
+                    type: string
+                description: Optional device volatile configuration keys
+                type: object
+                x-go-name: Volatile
+        title: AgentDeviceRemove represents the fields of an device removal request that needs to occur inside the VM agent.
+        type: object
+        x-go-package: github.com/canonical/lxd/shared/api
     AuthGroup:
         properties:
             description:

lxd-agent/api_1.0.go

@@ -29,6 +29,7 @@ var api10 = []APIEndpoint{
 	api10Cmd,
 	execCmd,
 	eventsCmd,
+	devicesCmd,
 	metricsCmd,
 	operationsCmd,
 	operationCmd,

lxd-agent/devices.go

@@ -0,0 +1,98 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/canonical/lxd/lxd/response"
+	"github.com/canonical/lxd/shared/api"
+)
+
+var devicesCmd = APIEndpoint{
+	Path: "devices",
+
+	Delete: APIEndpointAction{Handler: deviceDelete},
+}
+
+// deviceDelete handles the removal of a device from the VM agent.
+// e.g, if the device is a disk mount, this will cleanly unmount it and remove it if necessary.
+func deviceDelete(d *Daemon, r *http.Request) response.Response {
+	var device api.AgentDeviceRemove
+
+	err := json.NewDecoder(r.Body).Decode(&device)
+	if err != nil {
+		return response.InternalError(err)
+	}
+
+	// We only support disk devices for now
+	if device.Type != "disk" {
+		return response.BadRequest(fmt.Errorf("Device type %q not supported for removal within VM agent", device.Type))
+	}
+
+	targetPath := device.Config["path"]
+
+	if !filepath.IsAbs(targetPath) {
+		return response.SmartError(fmt.Errorf("The device path must be absolute: %q", device.Config["path"]))
+	}
+
+	file, err := os.Open("/proc/self/mountinfo")
+	if err != nil {
+		return response.SmartError(fmt.Errorf("Error opening /proc/self/mountinfo: %v", err))
+	}
+
+	defer file.Close()
+
+	var mountPoints []string
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		fields := strings.Fields(scanner.Text())
+		if len(fields) >= 10 && strings.HasPrefix(fields[4], strings.TrimSuffix(targetPath, "/")) {
+			mountPoints = append(mountPoints, fields[4])
+		}
+	}
+
+	err = scanner.Err()
+	if err != nil {
+		return response.SmartError(fmt.Errorf("Error reading /proc/self/mountinfo: %v", err))
+	}
+
+	if len(mountPoints) == 0 {
+		return response.SmartError(fmt.Errorf("No mount points found for %s", targetPath))
+	}
+
+	// Reverse the slice to unmount in reverse order.
+	// This is needed to unmount potential over-mounts first.
+	for i, j := 0, len(mountPoints)-1; i < j; i, j = i+1, j-1 {
+		mountPoints[i], mountPoints[j] = mountPoints[j], mountPoints[i]
+	}
+
+	for _, mountPoint := range mountPoints {
+		err = unix.Unmount(mountPoint, unix.MNT_DETACH)
+		if err != nil {
+			return response.SmartError(fmt.Errorf("Error unmounting %s: %v", mountPoint, err))
+		}
+	}
+
+	// Now that the unmount has occurred,
+	// check if we need to remove the target path.
+	if device.Volatile != nil {
+		path, ok := device.Volatile["last_state.created"]
+		if ok {
+			if targetPath == path {
+				err := os.Remove(targetPath)
+				if err != nil {
+					return response.SmartError(fmt.Errorf("Failed to remove last state file: %v", err))
+				}
+			}
+		}
+	}
+
+	return response.EmptySyncResponse
+}

lxd/device/disk.go

@@ -1956,6 +1956,7 @@ func (d *disk) Stop() (*deviceConfig.RunConfig, error) {
 
 	// Request an unmount of the device inside the instance.
 	runConf.Mounts = append(runConf.Mounts, deviceConfig.MountEntryItem{
+		DevName:    d.Name(),
 		TargetPath: relativeDestPath,
 	})
 

lxd/instance/drivers/driver_lxc.go

@@ -1481,7 +1481,7 @@ func (d *lxc) deviceStart(dev device.Device, instanceRunning bool) (*deviceConfi
 func (d *lxc) deviceStaticShiftMounts(mounts []deviceConfig.MountEntryItem) error {
 	idmapSet, err := d.CurrentIdmap()
 	if err != nil {
-		return fmt.Errorf("Failed to get idmap for device: %s", err)
+		return fmt.Errorf("Failed to get idmap for device: %w", err)
 	}
 
 	// If there is an idmap being applied and LXD not running in a user namespace then shift the
@@ -1688,6 +1688,15 @@ func (d *lxc) deviceDetachNIC(configCopy map[string]string, netIF []deviceConfig
 func (d *lxc) deviceHandleMounts(mounts []deviceConfig.MountEntryItem) error {
 	reverter := revert.New()
 	defer reverter.Fail()
+
+	// Connect to files API.
+	files, err := d.FileSFTP()
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = files.Close() }()
+
 	for _, mount := range mounts {
 		if mount.DevPath != "" {
 			flags := 0
@@ -1711,35 +1720,60 @@ func (d *lxc) deviceHandleMounts(mounts []deviceConfig.MountEntryItem) error {
 				}
 			}
 
-			// Mount it into the container.
-			err := d.insertMount(mount.DevPath, mount.TargetPath, mount.FSType, flags, idmapType)
+			_, err = files.Lstat(mount.TargetPath)
 			if err != nil {
-				return fmt.Errorf("Failed to add mount for device inside container: %s", err)
+				absTargetPath := mount.TargetPath
+				if !strings.HasPrefix(mount.TargetPath, "/") {
+					absTargetPath = fmt.Sprintf("/%s", mount.TargetPath)
+				}
+
+				err = d.deviceVolatileSetFunc(mount.DevName)(map[string]string{"last_state.created": absTargetPath}) // We want to store the absolute path.
+				if err != nil {
+					return fmt.Errorf("Error updating volatile for the device: %w", err)
+				}
 			}
-		} else {
-			relativeTargetPath := strings.TrimPrefix(mount.TargetPath, "/")
 
-			// Connect to files API.
-			files, err := d.FileSFTP()
+			// Mount it into the container.
+			err = d.insertMount(mount.DevPath, mount.TargetPath, mount.FSType, flags, idmapType)
 			if err != nil {
-				return err
+				return fmt.Errorf("Failed to add mount for device inside container: %w", err)
 			}
+		} else {
+			_, err = files.Lstat(mount.TargetPath)
+			if err == nil {
+				removeTargetFiles := false
+
+				// Check if the target path hasn't been created by LXD
+				mountConf := d.deviceVolatileGetFunc(mount.DevName)()
+				targetPath := mountConf["last_state.created"]
+				absMountTargetPath := mount.TargetPath
+				if !strings.HasPrefix(mount.TargetPath, "/") {
+					absMountTargetPath = fmt.Sprintf("/%s", mount.TargetPath)
+				}
 
-			reverter.Add(func() { _ = files.Close() })
+				if targetPath == absMountTargetPath {
+					removeTargetFiles = true
+				}
 
-			_, err = files.Lstat(relativeTargetPath)
-			if err == nil {
-				err := d.removeMount(mount.TargetPath)
+				err = d.removeMount(mount.TargetPath)
 				if err != nil {
-					return fmt.Errorf("Error unmounting the device path inside container: %s", err)
+					return fmt.Errorf("Error unmounting the device path inside container: %w", err)
 				}
 
-				err = files.Remove(relativeTargetPath)
-				if err != nil {
-					// Only warn here and don't fail as removing a directory
-					// mount may fail if there was already files inside
-					// directory before it was mouted over preventing delete.
-					d.logger.Warn("Could not remove the device path inside container", logger.Ctx{"err": err})
+				if removeTargetFiles {
+					err = files.Remove(mount.TargetPath)
+					if err != nil {
+						// Only warn here and don't fail as removing a directory
+						// mount may fail if there was already files inside
+						// directory before it was mouted over preventing delete.
+						d.logger.Warn("Could not remove the device path inside container", logger.Ctx{"err": err})
+					}
+				} else {
+					// Remove option from the device.
+					err = d.deviceVolatileSetFunc(mount.DevName)(map[string]string{"last_state.created": ""})
+					if err != nil {
+						return fmt.Errorf("Error updating volatile for the device: %w", err)
+					}
 				}
 			}
 
@@ -4111,7 +4145,7 @@ func (d *lxc) Update(args db.InstanceArgs, userRequested bool) error {
 	if args.Architecture != 0 {
 		_, err = osarch.ArchitectureName(args.Architecture)
 		if err != nil {
-			return fmt.Errorf("Invalid architecture id: %s", err)
+			return fmt.Errorf("Invalid architecture id: %w", err)
 		}
 	}
 
@@ -7545,12 +7579,12 @@ func (d *lxc) insertMountLXD(source, target, fstype string, flags int, mntnsPID
 	if shared.IsDir(source) {
 		tmpMount, err = os.MkdirTemp(d.ShmountsPath(), "lxdmount_")
 		if err != nil {
-			return fmt.Errorf("Failed to create shmounts path: %s", err)
+			return fmt.Errorf("Failed to create shmounts path: %w", err)
 		}
 	} else {
 		f, err := os.CreateTemp(d.ShmountsPath(), "lxdmount_")
 		if err != nil {
-			return fmt.Errorf("Failed to create shmounts path: %s", err)
+			return fmt.Errorf("Failed to create shmounts path: %w", err)
 		}
 
 		tmpMount = f.Name()
@@ -7562,7 +7596,7 @@ func (d *lxc) insertMountLXD(source, target, fstype string, flags int, mntnsPID
 	// Mount the filesystem
 	err = unix.Mount(source, tmpMount, fstype, uintptr(flags), "")
 	if err != nil {
-		return fmt.Errorf("Failed to setup temporary mount: %s", err)
+		return fmt.Errorf("Failed to setup temporary mount: %w", err)
 	}
 
 	defer func() { _ = unix.Unmount(tmpMount, unix.MNT_DETACH) }()
@@ -7804,7 +7838,7 @@ func (d *lxc) InsertSeccompUnixDevice(prefix string, m deviceConfig.Device, pid
 
 	dev, err := device.UnixDeviceCreate(d.state, idmapSet, d.DevicesPath(), prefix, m, true)
 	if err != nil {
-		return fmt.Errorf("Failed to setup device: %s", err)
+		return fmt.Errorf("Failed to setup device: %w", err)
 	}
 
 	devPath := dev.HostPath