diff --git a/pkg/sentry/kernel/kernel.go b/pkg/sentry/kernel/kernel.go index 05e71fb77a..f9d47122a2 100644 --- a/pkg/sentry/kernel/kernel.go +++ b/pkg/sentry/kernel/kernel.go @@ -370,6 +370,9 @@ type Kernel struct { // to be checkpointed. It's protected by checkpointMu. additionalCheckpointState map[any]any + // containerSpecs stores the specs of all containers. + containerSpecs map[any]any + // saver implements the Saver interface, which (as of writing) supports // asynchronous checkpointing. It's protected by checkpointMu. saver Saver `state:"nosave"` @@ -1903,6 +1906,29 @@ func (k *Kernel) PopCheckpointState(key any) any { return nil } +// AddContainerSpecs adds a key-value pair with key as container ID and value +// as spec for a container in the kernel. +func (k *Kernel) AddContainerSpecs(key, v any) { + k.checkpointMu.Lock() + defer k.checkpointMu.Unlock() + if k.containerSpecs == nil { + k.containerSpecs = make(map[any]any) + } + k.containerSpecs[key] = v +} + +// PopContainerSpecs pops all the key-value pairs from the container specs map. +func (k *Kernel) PopContainerSpecs() map[any]any { + k.checkpointMu.Lock() + defer k.checkpointMu.Unlock() + specs := make(map[any]any) + for key, val := range k.containerSpecs { + specs[key] = val + } + k.containerSpecs = nil + return specs +} + // HostMount returns the hostfs mount. func (k *Kernel) HostMount() *vfs.Mount { return k.hostMount diff --git a/runsc/boot/autosave.go b/runsc/boot/autosave.go index 7027313cae..3110e123a0 100644 --- a/runsc/boot/autosave.go +++ b/runsc/boot/autosave.go @@ -16,6 +16,7 @@ package boot import ( "bytes" + "encoding/json" "fmt" "gvisor.dev/gvisor/pkg/abi/linux" @@ -54,8 +55,22 @@ func getSaveOpts(l *Loader, k *kernel.Kernel, isResume bool) state.SaveOpts { return saveOpts } +func addContainerSpecsToKernel(l *Loader) { + l.mu.Lock() + defer l.mu.Unlock() + + for k, v := range l.containerSpecs { + data, err := json.Marshal(v) + if err != nil { + log.Warningf("json marshal error %v", err) + } + l.k.AddContainerSpecs(k, data) + } +} + func getTargetForSaveResume(l *Loader) func(k *kernel.Kernel) { return func(k *kernel.Kernel) { + addContainerSpecsToKernel(l) saveOpts := getSaveOpts(l, k, true /* isResume */) // Store the state file contents in a buffer for save-resume. // There is no need to verify the state file, we just need the @@ -74,6 +89,7 @@ func getTargetForSaveRestore(l *Loader, files []*fd.FD) func(k *kernel.Kernel) { var once sync.Once return func(k *kernel.Kernel) { once.Do(func() { + addContainerSpecsToKernel(l) saveOpts := getSaveOpts(l, k, false /* isResume */) saveOpts.Destination = files[0] if len(files) == 3 { diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go index acec716b43..a126438734 100644 --- a/runsc/boot/loader.go +++ b/runsc/boot/loader.go @@ -1941,3 +1941,10 @@ func (l *Loader) containerRuntimeState(cid string) ContainerRuntimeState { // Init process has stopped, but no one has called wait on it yet. return RuntimeStateStopped } + +// GetAllContainerSpecs returns all container specs. +func (l *Loader) GetAllContainerSpecs() map[string]*specs.Spec { + l.mu.Lock() + defer l.mu.Unlock() + return l.containerSpecs +} diff --git a/runsc/boot/restore.go b/runsc/boot/restore.go index ccf311e8d1..75b15ccba4 100644 --- a/runsc/boot/restore.go +++ b/runsc/boot/restore.go @@ -15,6 +15,7 @@ package boot import ( + "encoding/json" "errors" "fmt" "io" @@ -137,6 +138,14 @@ func createNetworkStackForRestore(l *Loader) (*stack.Stack, inet.Stack) { return nil, hostinet.NewStack() } +// Validate OCI specs before restoring the containers. +func validateSpecs(oldSpecs, newSpecs map[string]*specs.Spec) error { + if len(oldSpecs) != len(newSpecs) { + return fmt.Errorf("incorrect number of specs during checkpoint and restore") + } + return nil +} + func (r *restorer) restore(l *Loader) error { log.Infof("Starting to restore %d containers", len(r.containers)) @@ -226,6 +235,21 @@ func (r *restorer) restore(l *Loader) error { return err } + // Get the old specs from the loaded kernel and validate them with the + // new specs in the loader. + kSpecs := l.k.PopContainerSpecs() + oldSpecs := make(map[string]*specs.Spec) + for key, v := range kSpecs { + var s specs.Spec + if err := json.Unmarshal(v.([]byte), &s); err != nil { + return fmt.Errorf("json unmarshal error for specs %v", err) + } + oldSpecs[key.(string)] = &s + } + if err := validateSpecs(oldSpecs, l.containerSpecs); err != nil { + return err + } + // Since we have a new kernel we also must make a new watchdog. dogOpts := watchdog.DefaultOpts dogOpts.TaskTimeoutAction = l.root.conf.WatchdogAction @@ -311,6 +335,20 @@ func (r *restorer) restore(l *Loader) error { return nil } +func (l *Loader) saveSpecsInKernel() error { + l.mu.Lock() + defer l.mu.Unlock() + + for k, v := range l.containerSpecs { + data, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("json marshal error for specs %v", err) + } + l.k.AddContainerSpecs(k, data) + } + return nil +} + func (l *Loader) save(o *control.SaveOpts) (err error) { defer func() { // This closure is required to capture the final value of err. @@ -328,6 +366,10 @@ func (l *Loader) save(o *control.SaveOpts) (err error) { } o.Metadata["container_count"] = strconv.Itoa(l.containerCount()) + if err := l.saveSpecsInKernel(); err != nil { + return fmt.Errorf("saving specs in kernel failed with error: %v", err) + } + if err := preSaveImpl(l, o); err != nil { return err }