Generate correct output on NumCPU() when using cgroups2 (#11775)

Co-authored-by: Nicholas Orlowsky <nickorlow@nickorlow.com>

pkg/util/runtime/cpu_linux.go

@@ -36,15 +36,42 @@ import (
 //
 //	https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
 func NumCPU() int {
+	return NumCPUWithCustomPath("")
+}
+
+func NumCPUWithCustomPath(path string) int {
 	cpus := runtime.NumCPU()
 
-	cgroupPath, err := libcontainercgroups.FindCgroupMountpoint("", "cpu")
+	cgroupVersionCheckPath := path
+
+	if cgroupVersionCheckPath == "" {
+		cgroupVersionCheckPath = "/sys/fs/cgroup/"
+	}
+
+	cgroupVersion := GetCgroupVersion(cgroupVersionCheckPath)
+	cpuQuota := int64(-1)
+	cpuPeriod := int64(-1)
+
+	if cgroupVersion == 1 {
+		cgroupPath := ""
+		if path == "" {
+			cgroupPathRd, err := libcontainercgroups.FindCgroupMountpoint("", "cpu")
 			if err != nil {
 				return cpus
 			}
-
+			cgroupPath = cgroupPathRd
-	cpuQuota := readCgroupFileToInt64(cgroupPath, "cpu.cfs_quota_us")
+		} else {
-	cpuPeriod := readCgroupFileToInt64(cgroupPath, "cpu.cfs_period_us")
+			cgroupPath = path
+		}
+		cpuQuota = readCgroupFileToInt64(cgroupPath, "cpu.cfs_quota_us")
+		cpuPeriod = readCgroupFileToInt64(cgroupPath, "cpu.cfs_period_us")
+	} else if cgroupVersion == 2 {
+		cgroupPath := "/sys/fs/cgroup/"
+		if path != "" {
+			cgroupPath = path
+		}
+		cpuQuota, cpuPeriod = readCgroup2FileToInt64Tuple(cgroupPath, "cpu.max")
+	}
 
 	if cpuQuota == -1 || cpuPeriod == -1 {
 		return cpus
@@ -53,16 +80,66 @@ func NumCPU() int {
 	return int(math.Ceil(float64(cpuQuota) / float64(cpuPeriod)))
 }
 
-func readCgroupFileToInt64(cgroupPath, cgroupFile string) int64 {
+func GetCgroupVersion(cgroupPath string) int64 {
-	contents, err := os.ReadFile(filepath.Join(cgroupPath, cgroupFile))
+	// /sys/fs/cgroup/cgroup.controllers will not exist with cgroupsv1
-	if err != nil {
+	if _, err := os.Stat(filepath.Join(cgroupPath, "cgroup.controllers")); err == nil {
-		return -1
+		return 2
 	}
 
-	strValue := strings.TrimSpace(string(contents))
+	return 1
+}
+
+func readCgroup2StringToInt64Tuple(cgroupString string) (quota, period int64) {
+	// file contents looks like: $MAX $PERIOD
+	// $MAX can have value "max" indicating no limit
+	// it is possible for $PERIOD to be unset
+
+	values := strings.Fields(cgroupString)
+
+	if values[0] == "max" {
+		return -1, -1
+	}
+
+	cpuQuota, err := strconv.ParseInt(values[0], 10, 64)
+	if err != nil {
+		return -1, -1
+	}
+
+	if len(values) == 1 {
+		return cpuQuota, 100000
+	}
+
+	cpuPeriod, err := strconv.ParseInt(values[1], 10, 64)
+	if err != nil {
+		return -1, -1
+	}
+
+	return cpuQuota, cpuPeriod
+}
+
+func readCgroup2FileToInt64Tuple(cgroupPath, cgroupFile string) (quota, period int64) {
+	contents, err := os.ReadFile(filepath.Join(cgroupPath, cgroupFile))
+	if err != nil {
+		return -1, -1
+	}
+
+	return readCgroup2StringToInt64Tuple(string(contents))
+}
+
+func readCgroupStringToInt64(contents string) int64 {
+	strValue := strings.TrimSpace(contents)
 	if value, err := strconv.ParseInt(strValue, 10, 64); err == nil {
 		return value
 	}
 
 	return -1
 }
+
+func readCgroupFileToInt64(cgroupPath, cgroupFile string) int64 {
+	contents, err := os.ReadFile(filepath.Join(cgroupPath, cgroupFile))
+	if err != nil {
+		return -1
+	}
+
+	return readCgroupStringToInt64(string(contents))
+}

test/e2e/cgroups/cgroups.go

@@ -0,0 +1,115 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cgroups
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/stretchr/testify/assert"
+
+	"k8s.io/ingress-nginx/test/e2e/framework"
+
+	"k8s.io/ingress-nginx/pkg/util/runtime"
+)
+
+var _ = framework.IngressNginxDescribeSerial("[CGroups] cgroups", func() {
+	f := framework.NewDefaultFramework("cgroups")
+
+	ginkgo.BeforeEach(func() {
+		f.NewEchoDeployment()
+		f.NewSlowEchoDeployment()
+	})
+
+	ginkgo.It("detects cgroups version v1", func() {
+		cgroupPath := "/testing/sys/fs/cgroup/"
+		if err := os.MkdirAll(cgroupPath, os.ModePerm); err != nil {
+			log.Fatal(err)
+		}
+
+		quotaFile, err := os.Create(filepath.Join(cgroupPath, "cpu.cfs_quota_us"))
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		periodFile, err := os.Create(filepath.Join(cgroupPath, "cpu.cfs_period_us"))
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		_, err = quotaFile.WriteString("4")
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		err = quotaFile.Sync()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		_, err = periodFile.WriteString("2")
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		err = periodFile.Sync()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		assert.Equal(ginkgo.GinkgoT(), runtime.GetCgroupVersion(cgroupPath), int64(1))
+		assert.Equal(ginkgo.GinkgoT(), runtime.NumCPUWithCustomPath(cgroupPath), 2)
+
+		os.Remove(filepath.Join(cgroupPath, "cpu.cfs_quota_us"))
+		os.Remove(filepath.Join(cgroupPath, "cpu.cfs_period_us"))
+	})
+
+	ginkgo.It("detect cgroups version v2", func() {
+		cgroupPath := "/testing/sys/fs/cgroup/"
+		if err := os.MkdirAll(cgroupPath, os.ModePerm); err != nil {
+			log.Fatal(err)
+		}
+
+		_, err := os.Create(filepath.Join(cgroupPath, "cgroup.controllers"))
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		file, err := os.Create(filepath.Join(cgroupPath, "cpu.max"))
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		_, err = file.WriteString("4 2")
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		err = file.Sync()
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		assert.Equal(ginkgo.GinkgoT(), runtime.GetCgroupVersion(cgroupPath), int64(2))
+		assert.Equal(ginkgo.GinkgoT(), runtime.NumCPUWithCustomPath(cgroupPath), 2)
+
+		os.Remove(filepath.Join(cgroupPath, "cpu.max"))
+		os.Remove(filepath.Join(cgroupPath, "cgroup.controllers"))
+	})
+})

test/e2e/e2e.go

@@ -32,6 +32,7 @@ import (
 	_ "k8s.io/ingress-nginx/test/e2e/admission"
 	_ "k8s.io/ingress-nginx/test/e2e/annotations"
 	_ "k8s.io/ingress-nginx/test/e2e/annotations/modsecurity"
+	_ "k8s.io/ingress-nginx/test/e2e/cgroups"
 	_ "k8s.io/ingress-nginx/test/e2e/dbg"
 	_ "k8s.io/ingress-nginx/test/e2e/defaultbackend"
 	_ "k8s.io/ingress-nginx/test/e2e/endpointslices"