diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 1ed96710bbb0..97053f2ab0d4 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -19,6 +19,6 @@ func main() { } if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) { - logrus.Fatal(err) + logrus.Fatalf("Error: %v", err) } } diff --git a/cmd/k3s/main.go b/cmd/k3s/main.go index 0b3cad81d423..2b63b47b8d9d 100644 --- a/cmd/k3s/main.go +++ b/cmd/k3s/main.go @@ -83,7 +83,7 @@ func main() { } if err := app.Run(os.Args); err != nil && !errors.Is(err, context.Canceled) { - logrus.Fatal(err) + logrus.Fatalf("Error: %v", err) } } diff --git a/cmd/server/main.go b/cmd/server/main.go index 584ab486a814..193cdc0fd45f 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -80,6 +80,6 @@ func main() { } if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) { - logrus.Fatal(err) + logrus.Fatalf("Error: %v", err) } } diff --git a/go.mod b/go.mod index 842d06457c54..7288b394d0b9 100644 --- a/go.mod +++ b/go.mod @@ -157,6 +157,7 @@ require ( k8s.io/klog/v2 v2.130.1 k8s.io/kube-proxy v0.0.0 k8s.io/kubectl v0.31.1-rc.1 + k8s.io/kubelet v0.31.1 k8s.io/kubernetes v1.31.1 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 sigs.k8s.io/cri-tools v0.0.0-00010101000000-000000000000 @@ -477,7 +478,6 @@ require ( k8s.io/kube-controller-manager v0.0.0 // indirect k8s.io/kube-openapi v0.0.0-20240730131305-7a9a4e85957e // indirect k8s.io/kube-scheduler v0.0.0 // indirect - k8s.io/kubelet v0.31.1 // indirect k8s.io/metrics v0.0.0 // indirect k8s.io/mount-utils v0.31.1 // indirect k8s.io/pod-security-admission v0.0.0 // indirect diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 795618b03b83..21249f3c2426 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -534,6 +534,12 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N return nil, err } + // Ensure kubelet config dir exists + kubeletConfigDir := filepath.Join(envInfo.DataDir, "agent", "etc", "kubelet.conf.d") + if err := os.MkdirAll(kubeletConfigDir, 0700); err != nil { + return nil, err + } + nodeConfig := &config.Node{ Docker: envInfo.Docker, SELinux: envInfo.EnableSELinux, @@ -562,6 +568,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo) nodeConfig.AgentConfig.ClientCA = clientCAFile + nodeConfig.AgentConfig.KubeletConfigDir = kubeletConfigDir nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy nodeConfig.AgentConfig.KubeConfigK3sController = kubeconfigK3sController diff --git a/pkg/daemons/agent/agent.go b/pkg/daemons/agent/agent.go index 4c426ad81d98..36b945a70630 100644 --- a/pkg/daemons/agent/agent.go +++ b/pkg/daemons/agent/agent.go @@ -1,20 +1,36 @@ package agent import ( + "bytes" "context" + "fmt" "math/rand" + "net" "os" + "path/filepath" + "strings" "time" "github.com/k3s-io/k3s/pkg/agent/config" "github.com/k3s-io/k3s/pkg/agent/proxy" + "github.com/k3s-io/k3s/pkg/agent/util" daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/daemons/executor" + "github.com/k3s-io/k3s/pkg/version" + "github.com/otiai10/copy" + "github.com/pkg/errors" "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/component-base/logs" logsapi "k8s.io/component-base/logs/api/v1" + logsv1 "k8s.io/component-base/logs/api/v1" _ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration + kubeletconfig "k8s.io/kubelet/config/v1beta1" + "k8s.io/kubernetes/pkg/util/taints" + utilsnet "k8s.io/utils/net" + utilsptr "k8s.io/utils/ptr" + "sigs.k8s.io/yaml" ) func Agent(ctx context.Context, nodeConfig *daemonconfig.Node, proxy proxy.Proxy) error { @@ -24,7 +40,7 @@ func Agent(ctx context.Context, nodeConfig *daemonconfig.Node, proxy proxy.Proxy defer logs.FlushLogs() if err := startKubelet(ctx, &nodeConfig.AgentConfig); err != nil { - return err + return errors.Wrap(err, "failed to start kubelet") } go func() { @@ -46,9 +62,21 @@ func startKubeProxy(ctx context.Context, cfg *daemonconfig.Agent) error { } func startKubelet(ctx context.Context, cfg *daemonconfig.Agent) error { - argsMap := kubeletArgs(cfg) + argsMap, defaultConfig, err := kubeletArgsAndConfig(cfg) + if err != nil { + return errors.Wrap(err, "prepare default configuration drop-in") + } + + extraArgs, err := extractConfigArgs(cfg.KubeletConfigDir, cfg.ExtraKubeletArgs, defaultConfig) + if err != nil { + return errors.Wrap(err, "prepare user configuration drop-ins") + } + + if err := writeKubeletConfig(cfg.KubeletConfigDir, defaultConfig); err != nil { + return errors.Wrap(err, "generate default kubelet configuration drop-in") + } - args := daemonconfig.GetArgs(argsMap, cfg.ExtraKubeletArgs) + args := daemonconfig.GetArgs(argsMap, extraArgs) logrus.Infof("Running kubelet %s", daemonconfig.ArgString(args)) return executor.Kubelet(ctx, args) @@ -67,3 +95,170 @@ func ImageCredProvAvailable(cfg *daemonconfig.Agent) bool { } return true } + +// extractConfigArgs strips out any --config or --config-dir flags from the +// provided args list, and if set, copies the content of the file or dir into +// the target drop-in directory. +func extractConfigArgs(path string, extraArgs []string, config *kubeletconfig.KubeletConfiguration) ([]string, error) { + args := make([]string, 0, len(extraArgs)) + strippedArgs := map[string]string{} + var skipVal bool + for i := range extraArgs { + if skipVal { + skipVal = false + continue + } + + var val string + key := strings.TrimPrefix(extraArgs[i], "--") + if k, v, ok := strings.Cut(key, "="); ok { + // key=val pair + key = k + val = v + } else if len(extraArgs) > i+1 { + // key in this arg, value in next arg + val = extraArgs[i+1] + skipVal = true + } + + switch key { + case "config", "config-dir": + if val == "" { + return nil, fmt.Errorf("value required for kubelet-arg --%s", key) + } + strippedArgs[key] = val + default: + args = append(args, extraArgs[i]) + } + } + + // copy the config file into our managed config dir, unless its already in there + if strippedArgs["config"] != "" && !strings.HasPrefix(strippedArgs["config"], path) { + if err := util.CopyFile(strippedArgs["config"], filepath.Join(path, "10-cli-config.conf"), false); err != nil { + return nil, errors.Wrap(err, "copy config into managed drop-in dir") + } + } + // copy the config-dir into our managed config dir, unless its already in there + if strippedArgs["config-dir"] != "" && !strings.HasPrefix(strippedArgs["config-dir"], path) { + if err := copy.Copy(strippedArgs["config"], filepath.Join(path, "20-cli-config-dir"), copy.Options{PreserveOwner: true}); err != nil { + return nil, errors.Wrap(err, "copy config-dir into managed drop-in dir") + } + } + return args, nil +} + +// writeKubeletConfig marshals the provided KubeletConfiguration object into a +// drop-in config file in the target drop-in directory. +func writeKubeletConfig(path string, config *kubeletconfig.KubeletConfiguration) error { + b, err := yaml.Marshal(config) + if err != nil { + return err + } + + // replace resolvConf with resolverConfig until Kubernetes 1.32 + // ref: https://github.com/kubernetes/kubernetes/pull/127421 + b = bytes.ReplaceAll(b, []byte("resolvConf: "), []byte("resolverConfig: ")) + return os.WriteFile(filepath.Join(path, "00-"+version.Program+"-defaults.conf"), b, 0600) +} + +func defaultKubeletConfig(cfg *daemonconfig.Agent) (*kubeletconfig.KubeletConfiguration, error) { + bindAddress := "127.0.0.1" + isIPv6 := utilsnet.IsIPv6(net.ParseIP([]string{cfg.NodeIP}[0])) + if isIPv6 { + bindAddress = "::1" + } + + defaultConfig := &kubeletconfig.KubeletConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kubelet.config.k8s.io/v1beta1", + Kind: "KubeletConfiguration", + }, + CPUManagerReconcilePeriod: metav1.Duration{Duration: time.Second * 10}, + CgroupDriver: "cgroupfs", + ClusterDomain: cfg.ClusterDomain, + EvictionPressureTransitionPeriod: metav1.Duration{Duration: time.Minute * 5}, + FailSwapOn: utilsptr.To(false), + FileCheckFrequency: metav1.Duration{Duration: time.Second * 20}, + HTTPCheckFrequency: metav1.Duration{Duration: time.Second * 20}, + HealthzBindAddress: bindAddress, + ImageMinimumGCAge: metav1.Duration{Duration: time.Minute * 2}, + NodeStatusReportFrequency: metav1.Duration{Duration: time.Minute * 5}, + NodeStatusUpdateFrequency: metav1.Duration{Duration: time.Second * 10}, + ProtectKernelDefaults: cfg.ProtectKernelDefaults, + ReadOnlyPort: 0, + RuntimeRequestTimeout: metav1.Duration{Duration: time.Minute * 2}, + StreamingConnectionIdleTimeout: metav1.Duration{Duration: time.Hour * 4}, + SyncFrequency: metav1.Duration{Duration: time.Minute}, + VolumeStatsAggPeriod: metav1.Duration{Duration: time.Minute}, + EvictionHard: map[string]string{ + "imagefs.available": "5%", + "nodefs.available": "5%", + }, + EvictionMinimumReclaim: map[string]string{ + "imagefs.available": "10%", + "nodefs.available": "10%", + }, + Authentication: kubeletconfig.KubeletAuthentication{ + Anonymous: kubeletconfig.KubeletAnonymousAuthentication{ + Enabled: utilsptr.To(false), + }, + Webhook: kubeletconfig.KubeletWebhookAuthentication{ + Enabled: utilsptr.To(true), + CacheTTL: metav1.Duration{Duration: time.Minute * 2}, + }, + }, + Authorization: kubeletconfig.KubeletAuthorization{ + Mode: kubeletconfig.KubeletAuthorizationModeWebhook, + Webhook: kubeletconfig.KubeletWebhookAuthorization{ + CacheAuthorizedTTL: metav1.Duration{Duration: time.Minute * 5}, + CacheUnauthorizedTTL: metav1.Duration{Duration: time.Second * 30}, + }, + }, + Logging: logsv1.LoggingConfiguration{ + Format: "text", + Verbosity: logsv1.VerbosityLevel(cfg.VLevel), + FlushFrequency: logsv1.TimeOrMetaDuration{ + Duration: metav1.Duration{Duration: time.Second * 5}, + SerializeAsString: true, + }, + }, + } + + if cfg.ListenAddress != "" { + defaultConfig.Address = cfg.ListenAddress + } + + if cfg.ClientCA != "" { + defaultConfig.Authentication.X509.ClientCAFile = cfg.ClientCA + } + + if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" { + defaultConfig.TLSCertFile = cfg.ServingKubeletCert + defaultConfig.TLSPrivateKeyFile = cfg.ServingKubeletKey + } + + for _, addr := range cfg.ClusterDNSs { + defaultConfig.ClusterDNS = append(defaultConfig.ClusterDNS, addr.String()) + } + + if cfg.ResolvConf != "" { + defaultConfig.ResolverConfig = utilsptr.To(cfg.ResolvConf) + } + + if cfg.PodManifests != "" && defaultConfig.StaticPodPath == "" { + defaultConfig.StaticPodPath = cfg.PodManifests + } + if err := os.MkdirAll(defaultConfig.StaticPodPath, 0750); err != nil { + return nil, errors.Wrapf(err, "failed to create static pod manifest dir %s", defaultConfig.StaticPodPath) + } + + if t, _, err := taints.ParseTaints(cfg.NodeTaints); err != nil { + return nil, errors.Wrap(err, "failed to parse node taints") + } else { + defaultConfig.RegisterWithTaints = t + } + + logsv1.VModuleConfigurationPflag(&defaultConfig.Logging.VModule).Set(cfg.VModule) + + return defaultConfig, nil +} diff --git a/pkg/daemons/agent/agent_linux.go b/pkg/daemons/agent/agent_linux.go index ca7f94a529d0..1ce673d18ea3 100644 --- a/pkg/daemons/agent/agent_linux.go +++ b/pkg/daemons/agent/agent_linux.go @@ -5,7 +5,6 @@ package agent import ( "net" - "os" "path/filepath" "strconv" "strings" @@ -13,23 +12,25 @@ import ( "github.com/k3s-io/k3s/pkg/cgroups" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/util" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" - "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" + kubeletconfig "k8s.io/kubelet/config/v1beta1" utilsnet "k8s.io/utils/net" + utilsptr "k8s.io/utils/ptr" ) const socketPrefix = "unix://" -func createRootlessConfig(argsMap map[string]string, controllers map[string]bool) { +func createRootlessConfig(argsMap map[string]string, controllers map[string]bool) error { argsMap["feature-gates=KubeletInUserNamespace"] = "true" // "/sys/fs/cgroup" is namespaced cgroupfsWritable := unix.Access("/sys/fs/cgroup", unix.W_OK) == nil if controllers["cpu"] && controllers["pids"] && cgroupfsWritable { logrus.Info("cgroup v2 controllers are delegated for rootless.") - return + return nil } - logrus.Fatal("delegated cgroup v2 controllers are required for rootless.") + return errors.New("delegated cgroup v2 controllers are required for rootless") } func kubeProxyArgs(cfg *config.Agent) map[string]string { @@ -64,73 +65,49 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string { return argsMap } -func kubeletArgs(cfg *config.Agent) map[string]string { - bindAddress := "127.0.0.1" - if utilsnet.IsIPv6(net.ParseIP(cfg.NodeIP)) { - bindAddress = "::1" +// kubeletArgsAndConfig generates default kubelet args and configuration. +// Kubelet config is frustratingly split across deprecated CLI flags that raise warnings if you use them, +// and a structured configuration file that upstream does not provide a convienent way to initailize with default values. +// The defaults and our desired config also vary by OS. +func kubeletArgsAndConfig(cfg *config.Agent) (map[string]string, *kubeletconfig.KubeletConfiguration, error) { + defaultConfig, err := defaultKubeletConfig(cfg) + if err != nil { + return nil, nil, err } argsMap := map[string]string{ - "healthz-bind-address": bindAddress, - "read-only-port": "0", - "cluster-domain": cfg.ClusterDomain, - "kubeconfig": cfg.KubeConfigKubelet, - "eviction-hard": "imagefs.available<5%,nodefs.available<5%", - "eviction-minimum-reclaim": "imagefs.available=10%,nodefs.available=10%", - "fail-swap-on": "false", - "cgroup-driver": "cgroupfs", - "authentication-token-webhook": "true", - "anonymous-auth": "false", - "authorization-mode": modes.ModeWebhook, - } - if cfg.PodManifests != "" && argsMap["pod-manifest-path"] == "" { - argsMap["pod-manifest-path"] = cfg.PodManifests - } - if err := os.MkdirAll(argsMap["pod-manifest-path"], 0755); err != nil { - logrus.Errorf("Failed to mkdir %s: %v", argsMap["pod-manifest-path"], err) + "config-dir": cfg.KubeletConfigDir, + "kubeconfig": cfg.KubeConfigKubelet, } + if cfg.RootDir != "" { argsMap["root-dir"] = cfg.RootDir argsMap["cert-dir"] = filepath.Join(cfg.RootDir, "pki") } - if len(cfg.ClusterDNS) > 0 { - argsMap["cluster-dns"] = util.JoinIPs(cfg.ClusterDNSs) - } - if cfg.ResolvConf != "" { - argsMap["resolv-conf"] = cfg.ResolvConf - } if cfg.RuntimeSocket != "" { - argsMap["serialize-image-pulls"] = "false" + defaultConfig.SerializeImagePulls = utilsptr.To(false) + // note: this is a legacy cadvisor flag that the kubelet still exposes, but + // it must be set in order for cadvisor to pull stats properly. if strings.Contains(cfg.RuntimeSocket, "containerd") { argsMap["containerd"] = cfg.RuntimeSocket } // cadvisor wants the containerd CRI socket without the prefix, but kubelet wants it with the prefix if strings.HasPrefix(cfg.RuntimeSocket, socketPrefix) { - argsMap["container-runtime-endpoint"] = cfg.RuntimeSocket + defaultConfig.ContainerRuntimeEndpoint = cfg.RuntimeSocket } else { - argsMap["container-runtime-endpoint"] = socketPrefix + cfg.RuntimeSocket + defaultConfig.ContainerRuntimeEndpoint = socketPrefix + cfg.RuntimeSocket } } if cfg.ImageServiceSocket != "" { if strings.HasPrefix(cfg.ImageServiceSocket, socketPrefix) { - argsMap["image-service-endpoint"] = cfg.ImageServiceSocket + defaultConfig.ImageServiceEndpoint = cfg.ImageServiceSocket } else { - argsMap["image-service-endpoint"] = socketPrefix + cfg.ImageServiceSocket + defaultConfig.ImageServiceEndpoint = socketPrefix + cfg.ImageServiceSocket } } - if cfg.ListenAddress != "" { - argsMap["address"] = cfg.ListenAddress - } - if cfg.ClientCA != "" { - argsMap["anonymous-auth"] = "false" - argsMap["client-ca-file"] = cfg.ClientCA - } - if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" { - argsMap["tls-cert-file"] = cfg.ServingKubeletCert - argsMap["tls-private-key-file"] = cfg.ServingKubeletKey - } if cfg.NodeName != "" { argsMap["hostname-override"] = cfg.NodeName } + // If the embedded CCM is disabled, don't assume that dual-stack node IPs are safe. // When using an external CCM, the user wants dual-stack node IPs, they will need to set the node-ip kubelet arg directly. // This should be fine since most cloud providers have their own way of finding node IPs that doesn't depend on the kubelet @@ -141,35 +118,30 @@ func kubeletArgs(cfg *config.Agent) map[string]string { argsMap["node-ip"] = cfg.NodeIP } } else { + argsMap["cloud-provider"] = "external" // Cluster is using the embedded CCM, we know that the feature-gate will be enabled there as well. argsMap["feature-gates"] = util.AddFeatureGate(argsMap["feature-gates"], "CloudDualStackNodeIPs=true") if nodeIPs := util.JoinIPs(cfg.NodeIPs); nodeIPs != "" { argsMap["node-ip"] = util.JoinIPs(cfg.NodeIPs) } } + kubeletRoot, runtimeRoot, controllers := cgroups.CheckCgroups() + if !controllers["pids"] { + return nil, nil, errors.New("pids cgroup controller not found") + } if !controllers["cpu"] { logrus.Warn("Disabling CPU quotas due to missing cpu controller or cpu.cfs_period_us") - argsMap["cpu-cfs-quota"] = "false" - } - if !controllers["pids"] { - logrus.Fatal("pids cgroup controller not found") + defaultConfig.CPUCFSQuota = utilsptr.To(false) } if kubeletRoot != "" { - argsMap["kubelet-cgroups"] = kubeletRoot + defaultConfig.KubeletCgroups = kubeletRoot } if runtimeRoot != "" { argsMap["runtime-cgroups"] = runtimeRoot } argsMap["node-labels"] = strings.Join(cfg.NodeLabels, ",") - if len(cfg.NodeTaints) > 0 { - argsMap["register-with-taints"] = strings.Join(cfg.NodeTaints, ",") - } - - if !cfg.DisableCCM { - argsMap["cloud-provider"] = "external" - } if ImageCredProvAvailable(cfg) { logrus.Infof("Kubelet image credential provider bin dir and configuration file found.") @@ -178,25 +150,18 @@ func kubeletArgs(cfg *config.Agent) map[string]string { } if cfg.Rootless { - createRootlessConfig(argsMap, controllers) + if err := createRootlessConfig(argsMap, controllers); err != nil { + return nil, nil, err + } } if cfg.Systemd { - argsMap["cgroup-driver"] = "systemd" - } - - if cfg.ProtectKernelDefaults { - argsMap["protect-kernel-defaults"] = "true" + defaultConfig.CgroupDriver = "systemd" } if !cfg.DisableServiceLB { - argsMap["allowed-unsafe-sysctls"] = "net.ipv4.ip_forward,net.ipv6.conf.all.forwarding" + defaultConfig.AllowedUnsafeSysctls = []string{"net.ipv4.ip_forward", "net.ipv6.conf.all.forwarding"} } - if cfg.VLevel != 0 { - argsMap["v"] = strconv.Itoa(cfg.VLevel) - } - if cfg.VModule != "" { - argsMap["vmodule"] = cfg.VModule - } - return argsMap + + return argsMap, defaultConfig, nil } diff --git a/pkg/daemons/agent/agent_windows.go b/pkg/daemons/agent/agent_windows.go index 7bbf468eb6dd..c323ee04460c 100644 --- a/pkg/daemons/agent/agent_windows.go +++ b/pkg/daemons/agent/agent_windows.go @@ -5,15 +5,15 @@ package agent import ( "net" - "os" "path/filepath" "strings" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/util" "github.com/sirupsen/logrus" - "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" + kubeletconfig "k8s.io/kubelet/config/v1beta1" utilsnet "k8s.io/utils/net" + utilsptr "k8s.io/utils/ptr" ) const ( @@ -38,64 +38,42 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string { return argsMap } -func kubeletArgs(cfg *config.Agent) map[string]string { - bindAddress := "127.0.0.1" - _, IPv6only, _ := util.GetFirstString([]string{cfg.NodeIP}) - if IPv6only { - bindAddress = "::1" +// kubeletArgsAndConfig generates default kubelet args and configuration. +// Kubelet config is frustratingly split across deprecated CLI flags that raise warnings if you use them, +// and a structured configuration file that upstream does not provide a convienent way to initailize with default values. +// The defaults and our desired config also vary by OS. +func kubeletArgsAndConfig(cfg *config.Agent) (map[string]string, *kubeletconfig.KubeletConfiguration, error) { + defaultConfig, err := defaultKubeletConfig(cfg) + if err != nil { + return nil, nil, err } argsMap := map[string]string{ - "healthz-bind-address": bindAddress, - "read-only-port": "0", - "cluster-domain": cfg.ClusterDomain, - "kubeconfig": cfg.KubeConfigKubelet, - "eviction-hard": "imagefs.available<5%,nodefs.available<5%", - "eviction-minimum-reclaim": "imagefs.available=10%,nodefs.available=10%", - "fail-swap-on": "false", - "authentication-token-webhook": "true", - "anonymous-auth": "false", - "authorization-mode": modes.ModeWebhook, - } - if cfg.PodManifests != "" && argsMap["pod-manifest-path"] == "" { - argsMap["pod-manifest-path"] = cfg.PodManifests - } - if err := os.MkdirAll(argsMap["pod-manifest-path"], 0755); err != nil { - logrus.Errorf("Failed to mkdir %s: %v", argsMap["pod-manifest-path"], err) + "config-dir": cfg.KubeletConfigDir, + "kubeconfig": cfg.KubeConfigKubelet, } if cfg.RootDir != "" { argsMap["root-dir"] = cfg.RootDir argsMap["cert-dir"] = filepath.Join(cfg.RootDir, "pki") } - if len(cfg.ClusterDNS) > 0 { - argsMap["cluster-dns"] = util.JoinIPs(cfg.ClusterDNSs) - } - if cfg.ResolvConf != "" { - argsMap["resolv-conf"] = cfg.ResolvConf - } if cfg.RuntimeSocket != "" { - argsMap["serialize-image-pulls"] = "false" + defaultConfig.SerializeImagePulls = utilsptr.To(false) // cadvisor wants the containerd CRI socket without the prefix, but kubelet wants it with the prefix if strings.HasPrefix(cfg.RuntimeSocket, socketPrefix) { - argsMap["container-runtime-endpoint"] = cfg.RuntimeSocket + defaultConfig.ContainerRuntimeEndpoint = cfg.RuntimeSocket } else { - argsMap["container-runtime-endpoint"] = socketPrefix + cfg.RuntimeSocket + defaultConfig.ContainerRuntimeEndpoint = socketPrefix + cfg.RuntimeSocket } } - if cfg.ListenAddress != "" { - argsMap["address"] = cfg.ListenAddress - } - if cfg.ClientCA != "" { - argsMap["anonymous-auth"] = "false" - argsMap["client-ca-file"] = cfg.ClientCA - } - if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" { - argsMap["tls-cert-file"] = cfg.ServingKubeletCert - argsMap["tls-private-key-file"] = cfg.ServingKubeletKey + if cfg.ImageServiceSocket != "" { + if strings.HasPrefix(cfg.ImageServiceSocket, socketPrefix) { + defaultConfig.ImageServiceEndpoint = cfg.ImageServiceSocket + } else { + defaultConfig.ImageServiceEndpoint = socketPrefix + cfg.ImageServiceSocket + } } if cfg.NodeName != "" { argsMap["hostname-override"] = cfg.NodeName } - // If the embedded CCM is disabled, don't assume that dual-stack node IPs are safe. // When using an external CCM, the user wants dual-stack node IPs, they will need to set the node-ip kubelet arg directly. // This should be fine since most cloud providers have their own way of finding node IPs that doesn't depend on the kubelet @@ -106,6 +84,7 @@ func kubeletArgs(cfg *config.Agent) map[string]string { argsMap["node-ip"] = cfg.NodeIP } } else { + argsMap["cloud-provider"] = "external" // Cluster is using the embedded CCM, we know that the feature-gate will be enabled there as well. argsMap["feature-gates"] = util.AddFeatureGate(argsMap["feature-gates"], "CloudDualStackNodeIPs=true") if nodeIPs := util.JoinIPs(cfg.NodeIPs); nodeIPs != "" { @@ -114,13 +93,6 @@ func kubeletArgs(cfg *config.Agent) map[string]string { } argsMap["node-labels"] = strings.Join(cfg.NodeLabels, ",") - if len(cfg.NodeTaints) > 0 { - argsMap["register-with-taints"] = strings.Join(cfg.NodeTaints, ",") - } - - if !cfg.DisableCCM { - argsMap["cloud-provider"] = "external" - } if ImageCredProvAvailable(cfg) { logrus.Infof("Kubelet image credential provider bin dir and configuration file found.") @@ -128,9 +100,5 @@ func kubeletArgs(cfg *config.Agent) map[string]string { argsMap["image-credential-provider-config"] = cfg.ImageCredProvConfig } - if cfg.ProtectKernelDefaults { - argsMap["protect-kernel-defaults"] = "true" - } - - return argsMap + return argsMap, defaultConfig, nil } diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 93e354e1962c..9616c6136eab 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -116,6 +116,7 @@ type Agent struct { ClusterDomain string ResolvConf string RootDir string + KubeletConfigDir string KubeConfigKubelet string KubeConfigKubeProxy string KubeConfigK3sController string diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index 34eafdd7c070..7f735d57e057 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -32,7 +32,7 @@ import ( "github.com/k3s-io/k3s/pkg/version" "github.com/k3s-io/kine/pkg/client" endpoint2 "github.com/k3s-io/kine/pkg/endpoint" - cp "github.com/otiai10/copy" + "github.com/otiai10/copy" "github.com/pkg/errors" certutil "github.com/rancher/dynamiclistener/cert" controllerv1 "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1" @@ -997,7 +997,7 @@ func (e *ETCD) StartEmbeddedTemporary(ctx context.Context) error { client.Close() }() - if err := cp.Copy(etcdDataDir, tmpDataDir, cp.Options{PreserveOwner: true}); err != nil { + if err := copy.Copy(etcdDataDir, tmpDataDir, copy.Options{PreserveOwner: true}); err != nil { return err } diff --git a/tests/integration/integration.go b/tests/integration/integration.go index 6f6745dbcb65..3841ac19d40f 100644 --- a/tests/integration/integration.go +++ b/tests/integration/integration.go @@ -130,33 +130,24 @@ func K3sServerArgs() []string { // K3sDefaultDeployments checks if the default deployments for K3s are ready, otherwise returns an error func K3sDefaultDeployments() error { - return CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}) + return CheckDeployments(metav1.NamespaceSystem, []string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}) } // CheckDeployments checks if the provided list of deployments are ready, otherwise returns an error -func CheckDeployments(deployments []string) error { - - deploymentSet := make(map[string]bool) - for _, d := range deployments { - deploymentSet[d] = false - } - +func CheckDeployments(namespace string, deployments []string) error { client, err := k8sClient() if err != nil { return err } - deploymentList, err := client.AppsV1().Deployments("").List(context.Background(), metav1.ListOptions{}) - if err != nil { - return err - } - for _, deployment := range deploymentList.Items { - if _, ok := deploymentSet[deployment.Name]; ok && deployment.Status.ReadyReplicas == deployment.Status.Replicas { - deploymentSet[deployment.Name] = true + + for _, deploymentName := range deployments { + deployment, err := client.AppsV1().Deployments(namespace).Get(context.Background(), deploymentName, metav1.GetOptions{}) + if err != nil { + return err } - } - for d, found := range deploymentSet { - if !found { - return fmt.Errorf("failed to deploy %s", d) + if deployment.Status.ReadyReplicas != deployment.Status.Replicas || deployment.Status.AvailableReplicas != deployment.Status.Replicas { + return fmt.Errorf("deployment %s not ready: replicas=%d readyReplicas=%d availableReplicas=%d", + deploymentName, deployment.Status.Replicas, deployment.Status.ReadyReplicas, deployment.Status.AvailableReplicas) } } diff --git a/tests/integration/startup/startup_int_test.go b/tests/integration/startup/startup_int_test.go index e0f23f916db3..bc339f8d4a57 100644 --- a/tests/integration/startup/startup_int_test.go +++ b/tests/integration/startup/startup_int_test.go @@ -124,7 +124,7 @@ var _ = Describe("startup tests", Ordered, func() { }) It("has the default pods without traefik deployed", func() { Eventually(func() error { - return testutil.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server"}) + return testutil.CheckDeployments("kube-system", []string{"coredns", "local-path-provisioner", "metrics-server"}) }, "90s", "10s").Should(Succeed()) nodes, err := testutil.ParseNodes() Expect(err).NotTo(HaveOccurred())