Kubernetes问题调查: failed to get cgroup stats for /systemd/system.slice

// pkg/kubelet/server/stats/summary.go
// Get provides a new Summary with the stats from Kubelet.
func (sp *summaryProviderImpl) Get() (*statsapi.Summary, error) {
	...省略...
	systemContainers := map[string]string{
		statsapi.SystemContainerKubelet: nodeConfig.KubeletCgroupsName,
		statsapi.SystemContainerRuntime: nodeConfig.RuntimeCgroupsName,
		statsapi.SystemContainerMisc:    nodeConfig.SystemCgroupsName,
	}
	for sys, name := range systemContainers {
		// skip if cgroup name is undefined (not all system containers are required)
		if name == "" {
			continue
		}
		s, _, err := sp.provider.GetCgroupStats(name)
		if err != nil {
			glog.Errorf("Failed to get system container stats for %q: %v", name, err)
			continue
		}
		// System containers don't have a filesystem associated with them.
		s.Logs, s.Rootfs = nil, nil
		s.Name = sys
		nodeStats.SystemContainers = append(nodeStats.SystemContainers, *s)
	}
	...省略...
	}

// cmd/kubelet/app/server.go: 418
func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) (err error) {
	...
	if kubeDeps.CAdvisorInterface == nil {
		imageFsInfoProvider := cadvisor.NewImageFsInfoProvider(s.ContainerRuntime, s.RemoteRuntimeEndpoint)
		kubeDeps.CAdvisorInterface, err = cadvisor.New(s.Address, uint(s.CAdvisorPort), imageFsInfoProvider, s.RootDirectory, cadvisor.UsingLegacyCadvisorStats(s.ContainerRuntime, s.RemoteRuntimeEndpoint))
		if err != nil {
			return err
		}
	}

// Copyright (C) 2019 lijiaocn <[email protected]>
//
// Distributed under terms of the GPL license.

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/golang/glog"
	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
	"time"
)

type CmdLine struct {
	ContainerName string
}

var cmdline CmdLine

func init() {
	flag.StringVar(&cmdline.ContainerName, "name", "/", "container name")
}

func main() {
	flag.Parse()

	ContainerRuntime := "docker"
	RemoteRuntimeEndpoint := ""
	imagefs := cadvisor.NewImageFsInfoProvider(ContainerRuntime, RemoteRuntimeEndpoint)
	usingLegacyStats := cadvisor.UsingLegacyCadvisorStats(ContainerRuntime, RemoteRuntimeEndpoint)
	ca, err := cadvisor.New("127.0.0.1", 4000, imagefs, "/var/lib/kubelet", usingLegacyStats)
	if err != nil {
		glog.Error(err)
	}

	if err := ca.Start(); err != nil {
		glog.Error(err)
	}

	options := cadvisorapiv2.RequestOptions{
		IdType:    cadvisorapiv2.TypeName,
		Count:     2, // 2 samples are needed to compute "instantaneous" CPU
		Recursive: true,   //设置为true，递归查找，深入到目录中
	}

	machineInfo, err := ca.MachineInfo()
	if err != nil {
		glog.Error(err.Error())
	}
	bytes, err := json.Marshal(&machineInfo)
	fmt.Printf("machineInfo: %s\n", bytes)

	rootfsInfo, err := ca.RootFsInfo()
	if err != nil {
		glog.Error(err.Error())
	}
	bytes, err = json.Marshal(&rootfsInfo)
	fmt.Printf("rootfsInfo: %s\n", bytes)

	infoMap, err := ca.ContainerInfoV2(cmdline.ContainerName, options)
	if err != nil {
		glog.Error(err.Error())
	}

	if infoMap == nil {
		glog.Error("infoMap is null")
	}

	for name, value := range infoMap {
		fmt.Printf("%s: %s  %s\n", name, value.Spec.Namespace, value.Spec.Image)
	}

	time.Sleep(600 * time.Second)
}