From e866b6ee1cc9106818b6ff22a942af6e9f4a5284 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 22 Nov 2018 15:44:30 +0200 Subject: [PATCH] Implement RDT detection in go Get rid of the dependency on intel-cmt-cat library and rdt helper binaries written in C. Significantly simplifies the build procedure. Implements minimal support (in assembler) for getting the raw data from the CPUID instruction. Also, implement a stub so that the code works on other architectures than amd64, too. --- Dockerfile | 17 ---- rdt-discovery/Makefile | 31 ------ rdt-discovery/l2-allocation-discovery.c | 22 ----- rdt-discovery/l3-allocation-discovery.c | 24 ----- .../mem-bandwidth-allocation-discovery.c | 22 ----- rdt-discovery/monitoring-cmt-discovery.c | 26 ----- rdt-discovery/monitoring-discovery.c | 22 ----- rdt-discovery/monitoring-mbm-discovery.c | 26 ----- source/rdt/rdt.go | 51 +--------- source/rdt/rdt_amd64.go | 98 +++++++++++++++++++ source/rdt/rdt_amd64.s | 27 +++++ source/rdt/rdt_stub.go | 23 +++++ 12 files changed, 151 insertions(+), 238 deletions(-) delete mode 100644 rdt-discovery/Makefile delete mode 100644 rdt-discovery/l2-allocation-discovery.c delete mode 100644 rdt-discovery/l3-allocation-discovery.c delete mode 100644 rdt-discovery/mem-bandwidth-allocation-discovery.c delete mode 100644 rdt-discovery/monitoring-cmt-discovery.c delete mode 100644 rdt-discovery/monitoring-discovery.c delete mode 100644 rdt-discovery/monitoring-mbm-discovery.c create mode 100644 source/rdt/rdt_amd64.go create mode 100644 source/rdt/rdt_amd64.s create mode 100644 source/rdt/rdt_stub.go diff --git a/Dockerfile b/Dockerfile index 22951132e..8e336bca2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,22 +5,8 @@ ADD . /go/src/sigs.k8s.io/node-feature-discovery WORKDIR /go/src/sigs.k8s.io/node-feature-discovery -ENV CMT_CAT_VERSION="v1.2.0" - ARG NFD_VERSION -RUN case $(dpkg --print-architecture) in \ - arm64) \ - echo "skip rdt on Arm64 platform" \ - ;; \ - *) \ - git clone --depth 1 -b $CMT_CAT_VERSION https://github.com/intel/intel-cmt-cat.git && \ - make -C intel-cmt-cat/lib install && \ - make -C rdt-discovery && \ - make -C rdt-discovery install \ - ;; \ - esac - RUN go get github.com/Masterminds/glide RUN glide install --strip-vendor RUN go install \ @@ -34,10 +20,7 @@ RUN go test . # Create production image for running node feature discovery FROM debian:stretch-slim -COPY --from=builder /usr/local/bin /usr/local/bin -COPY --from=builder /usr/local/lib /usr/local/lib COPY --from=builder /etc/kubernetes/node-feature-discovery /etc/kubernetes/node-feature-discovery -RUN ldconfig COPY --from=builder /go/bin/node-feature-discovery /usr/bin/node-feature-discovery ENTRYPOINT ["/usr/bin/node-feature-discovery"] diff --git a/rdt-discovery/Makefile b/rdt-discovery/Makefile deleted file mode 100644 index 238997d0d..000000000 --- a/rdt-discovery/Makefile +++ /dev/null @@ -1,31 +0,0 @@ -CC=gcc -LIBDIR=/usr/local/lib -INCDIR=../intel-cmt-cat/lib/ - -PREFIX ?= /usr/local -BINDIR ?= $(PREFIX)/bin - -LDFLAGS=-L$(LIBDIR) -LDLIBS=-lpqos -CFLAGS=-I$(INCDIR) - -BINARIES = mon-discovery \ - mon-cmt-discovery \ - mon-mbm-discovery \ - l3-alloc-discovery \ - l2-alloc-discovery \ - mem-bandwidth-alloc-discovery - -default: - $(MAKE) all -all: - $(CC) $(CFLAGS) -o mon-discovery monitoring-discovery.c $(LDFLAGS) $(LDLIBS) - $(CC) $(CFLAGS) -o mon-cmt-discovery monitoring-cmt-discovery.c $(LDFLAGS) $(LDLIBS) - $(CC) $(CFLAGS) -o mon-mbm-discovery monitoring-mbm-discovery.c $(LDFLAGS) $(LDLIBS) - $(CC) $(CFLAGS) -o l3-alloc-discovery l3-allocation-discovery.c $(LDFLAGS) $(LDLIBS) - $(CC) $(CFLAGS) -o l2-alloc-discovery l2-allocation-discovery.c $(LDFLAGS) $(LDLIBS) - $(CC) $(CFLAGS) -o mem-bandwidth-alloc-discovery mem-bandwidth-allocation-discovery.c $(LDFLAGS) $(LDLIBS) - -install: - install -d $(BINDIR) - install -t $(BINDIR) $(BINARIES) diff --git a/rdt-discovery/l2-allocation-discovery.c b/rdt-discovery/l2-allocation-discovery.c deleted file mode 100644 index 37058dd4b..000000000 --- a/rdt-discovery/l2-allocation-discovery.c +++ /dev/null @@ -1,22 +0,0 @@ -#include -#include -#include "machine.h" - -int main(int argc, char *argv[]) { - struct cpuid_out res; - - // Logic below from https://github.com/intel/intel-cmt-cat/blob/master/lib/cap.c - lcpuid(0x7, 0x0, &res); - if (!(res.ebx & (1 << 15))) { - return EXIT_FAILURE; - } - else { - lcpuid(0x10, 0x0, &res); - if (!(res.ebx & (1 << 2))) { - return EXIT_FAILURE; - } - } - - // If we are here, then L2 cache allocation capability is available. - return EXIT_SUCCESS; -} diff --git a/rdt-discovery/l3-allocation-discovery.c b/rdt-discovery/l3-allocation-discovery.c deleted file mode 100644 index b402a77c1..000000000 --- a/rdt-discovery/l3-allocation-discovery.c +++ /dev/null @@ -1,24 +0,0 @@ -#include -#include -#include "machine.h" - -int main(int argc, char *argv[]) { - struct cpuid_out res; - - // Logic below from https://github.com/intel/intel-cmt-cat/blob/master/lib/cap.c - // TODO(balajismaniam): Implement L3 CAT detection using brand string and MSR probing if - // not detected using cpuid - lcpuid(0x7, 0x0, &res); - if (!(res.ebx & (1 << 15))) { - return EXIT_FAILURE; - } - else { - lcpuid(0x10, 0x0, &res); - if (!(res.ebx & (1 << 1))) { - return EXIT_FAILURE; - } - } - - // If we are here, then L3 cache allocation capability is available. - return EXIT_SUCCESS; -} diff --git a/rdt-discovery/mem-bandwidth-allocation-discovery.c b/rdt-discovery/mem-bandwidth-allocation-discovery.c deleted file mode 100644 index 5d8ebb514..000000000 --- a/rdt-discovery/mem-bandwidth-allocation-discovery.c +++ /dev/null @@ -1,22 +0,0 @@ -#include -#include -#include "machine.h" - -int main(int argc, char *argv[]) { - struct cpuid_out res; - - // Logic below from https://github.com/intel/intel-cmt-cat/blob/master/lib/cap.c - lcpuid(0x7, 0x0, &res); - if (!(res.ebx & (1 << 15))) { - return EXIT_FAILURE; - } - else { - lcpuid(0x10, 0x0, &res); - if (!(res.ebx & (1 << 3))) { - return EXIT_FAILURE; - } - } - - // If we are here, then Memory Bandwidth Allocation capability is available. - return EXIT_SUCCESS; -} diff --git a/rdt-discovery/monitoring-cmt-discovery.c b/rdt-discovery/monitoring-cmt-discovery.c deleted file mode 100644 index 430f10fb4..000000000 --- a/rdt-discovery/monitoring-cmt-discovery.c +++ /dev/null @@ -1,26 +0,0 @@ -#include -#include -#include "machine.h" - -int main(int argc, char *argv[]) { - struct cpuid_out res; - - // Logic below from https://github.com/intel/intel-cmt-cat/blob/master/lib/cap.c - lcpuid(0x7, 0x0, &res); - if (!(res.ebx & (1 << 12))) { - return EXIT_FAILURE; - } - // check for overall monitoring capability first - lcpuid(0xf, 0x0, &res); - if (!(res.edx & (1 << 1))) { - return EXIT_FAILURE; - } - // check for more detailed capability, CMT monitoring - lcpuid(0xf, 0x1, &res); - if (!(res.edx & (1 << 0))) { - return EXIT_FAILURE; - } - - // If we are here, then CMT cache monitoring capability is available. - return EXIT_SUCCESS; -} diff --git a/rdt-discovery/monitoring-discovery.c b/rdt-discovery/monitoring-discovery.c deleted file mode 100644 index 65a596ace..000000000 --- a/rdt-discovery/monitoring-discovery.c +++ /dev/null @@ -1,22 +0,0 @@ -#include -#include -#include "machine.h" - -int main(int argc, char *argv[]) { - struct cpuid_out res; - - // Logic below from https://github.com/intel/intel-cmt-cat/blob/master/lib/cap.c - lcpuid(0x7, 0x0, &res); - if (!(res.ebx & (1 << 12))) { - return EXIT_FAILURE; - } - else { - lcpuid(0xf, 0x0, &res); - if (!(res.edx & (1 << 1))) { - return EXIT_FAILURE; - } - } - - // If we are here, then cache monitoring capability is available. - return EXIT_SUCCESS; -} diff --git a/rdt-discovery/monitoring-mbm-discovery.c b/rdt-discovery/monitoring-mbm-discovery.c deleted file mode 100644 index cd2c0124b..000000000 --- a/rdt-discovery/monitoring-mbm-discovery.c +++ /dev/null @@ -1,26 +0,0 @@ -#include -#include -#include "machine.h" - -int main(int argc, char *argv[]) { - struct cpuid_out res; - - // Logic below from https://github.com/intel/intel-cmt-cat/blob/master/lib/cap.c - lcpuid(0x7, 0x0, &res); - if (!(res.ebx & (1 << 12))) { - return EXIT_FAILURE; - } - // check for overall monitoring capability first - lcpuid(0xf, 0x0, &res); - if (!(res.edx & (1 << 1))) { - return EXIT_FAILURE; - } - // check for more detailed capability, MBM monitoring - lcpuid(0xf, 0x1, &res); - if ((res.edx & (3 << 1)) != (3 << 1)) { - return EXIT_FAILURE; - } - - // If we are here, then MBM cache monitoring capability is available. - return EXIT_SUCCESS; -} diff --git a/source/rdt/rdt.go b/source/rdt/rdt.go index 1d97d2321..122e527ee 100644 --- a/source/rdt/rdt.go +++ b/source/rdt/rdt.go @@ -17,9 +17,6 @@ limitations under the License. package rdt import ( - "os/exec" - - "github.com/golang/glog" "sigs.k8s.io/node-feature-discovery/source" ) @@ -33,52 +30,10 @@ func (s Source) Name() string { return "rdt" } func (s Source) Discover() (source.Features, error) { features := source.Features{} - cmd := exec.Command("bash", "-c", "mon-discovery") - if err := cmd.Run(); err != nil { - glog.Errorf("support for RDT monitoring was not detected: %v", err) - } else { - // RDT monitoring detected. - features["RDTMON"] = true - } + rdtFeatures := discoverRDT() - cmd = exec.Command("bash", "-c", "mon-cmt-discovery") - if err := cmd.Run(); err != nil { - glog.Errorf("support for RDT CMT monitoring was not detected: %v", err) - } else { - // RDT CMT monitoring detected. - features["RDTCMT"] = true - } - - cmd = exec.Command("bash", "-c", "mon-mbm-discovery") - if err := cmd.Run(); err != nil { - glog.Errorf("support for RDT MBM monitoring was not detected: %v", err) - } else { - // RDT MBM monitoring detected. - features["RDTMBM"] = true - } - - cmd = exec.Command("bash", "-c", "l3-alloc-discovery") - if err := cmd.Run(); err != nil { - glog.Errorf("support for RDT L3 allocation was not detected: %v", err) - } else { - // RDT L3 cache allocation detected. - features["RDTL3CA"] = true - } - - cmd = exec.Command("bash", "-c", "l2-alloc-discovery") - if err := cmd.Run(); err != nil { - glog.Errorf("support for RDT L2 allocation was not detected: %v", err) - } else { - // RDT L2 cache allocation detected. - features["RDTL2CA"] = true - } - - cmd = exec.Command("bash", "-c", "mem-bandwidth-alloc-discovery") - if err := cmd.Run(); err != nil { - glog.Errorf("support for RDT Memory bandwidth allocation was not detected: %v", err) - } else { - // RDT Memory bandwidth allocation detected. - features["RDTMBA"] = true + for _, f := range rdtFeatures { + features[f] = true } return features, nil diff --git a/source/rdt/rdt_amd64.go b/source/rdt/rdt_amd64.go new file mode 100644 index 000000000..223ecfeb6 --- /dev/null +++ b/source/rdt/rdt_amd64.go @@ -0,0 +1,98 @@ +// +build amd64 + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rdt + +type CpuidRet struct { + EAX, EBX, ECX, EDX uint32 +} + +const ( + // CPUID EAX input values + LEAF_EXT_FEATURE_FLAGS = 0x07 + LEAF_RDT_MONITORING = 0x0f + LEAF_RDT_ALLOCATION = 0x10 + + // CPUID ECX input values + RDT_MONITORING_SUBLEAF_L3 = 1 + + // CPUID bitmasks + EXT_FEATURE_FLAGS_EBX_RDT_M = 1 << 12 + EXT_FEATURE_FLAGS_EBX_RDT_A = 1 << 15 + RDT_MONITORING_EDX_L3_MONITORING = 1 << 1 + RDT_MONITORING_SUBLEAF_L3_EDX_L3_OCCUPANCY_MONITORING = 1 << 0 + RDT_MONITORING_SUBLEAF_L3_EDX_L3_TOTAL_BANDWIDTH_MONITORING = 1 << 1 + RDT_MONITORING_SUBLEAF_L3_EDX_L3_LOCAL_BANDWIDTH_MONITORING = 1 << 2 + RDT_ALLOCATION_EBX_L3_CACHE_ALLOCATION = 1 << 1 + RDT_ALLOCATION_EBX_L2_CACHE_ALLOCATION = 1 << 2 + RDT_ALLOCATION_EBX_MEMORY_BANDWIDTH_ALLOCATION = 1 << 3 +) + +func discoverRDT() []string { + features := []string{} + + // Read cpuid information + extFeatures := cpuid(LEAF_EXT_FEATURE_FLAGS, 0) + rdtMonitoring := cpuid(LEAF_RDT_MONITORING, 0) + rdtL3Monitoring := cpuid(LEAF_RDT_MONITORING, RDT_MONITORING_SUBLEAF_L3) + rdtAllocation := cpuid(LEAF_RDT_ALLOCATION, 0) + + // Detect RDT monitoring capabilities + if extFeatures.EBX&EXT_FEATURE_FLAGS_EBX_RDT_M != 0 { + if rdtMonitoring.EDX&RDT_MONITORING_EDX_L3_MONITORING != 0 { + // Monitoring is supported + features = append(features, "RDTMON") + + // Cache Monitoring Technology (L3 occupancy monitoring) + if rdtL3Monitoring.EDX&RDT_MONITORING_SUBLEAF_L3_EDX_L3_OCCUPANCY_MONITORING != 0 { + features = append(features, "RDTCMT") + } + // Memore Bandwidth Monitoring (L3 local&total bandwidth monitoring) + if rdtL3Monitoring.EDX&RDT_MONITORING_SUBLEAF_L3_EDX_L3_TOTAL_BANDWIDTH_MONITORING != 0 && + rdtL3Monitoring.EDX&RDT_MONITORING_SUBLEAF_L3_EDX_L3_LOCAL_BANDWIDTH_MONITORING != 0 { + features = append(features, "RDTMBM") + } + } + } + + // Detect RDT allocation capabilities + if extFeatures.EBX&EXT_FEATURE_FLAGS_EBX_RDT_A != 0 { + // L3 Cache Allocation + if rdtAllocation.EBX&RDT_ALLOCATION_EBX_L3_CACHE_ALLOCATION != 0 { + features = append(features, "RDTL3CA") + } + // L2 Cache Allocation + if rdtAllocation.EBX&RDT_ALLOCATION_EBX_L2_CACHE_ALLOCATION != 0 { + features = append(features, "RDTL2CA") + } + // Memory Bandwidth Allocation + if rdtAllocation.EBX&RDT_ALLOCATION_EBX_MEMORY_BANDWIDTH_ALLOCATION != 0 { + features = append(features, "RDTMBA") + } + } + + return features +} + +func cpuid(eax, ecx uint32) CpuidRet { + r := CpuidRet{} + r.EAX, r.EBX, r.ECX, r.EDX = cpuidAsm(eax, ecx) + return r +} + +func cpuidAsm(eax_arg, ecx_arg uint32) (eax, ebx, ecx, edx uint32) diff --git a/source/rdt/rdt_amd64.s b/source/rdt/rdt_amd64.s new file mode 100644 index 000000000..83ac812c1 --- /dev/null +++ b/source/rdt/rdt_amd64.s @@ -0,0 +1,27 @@ +// +build amd64 + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +TEXT ·cpuidAsm(SB), 4, $0 // 4 = NOSPLIT + MOVL eax_arg+0(FP), AX + MOVL ecx_arg+4(FP), CX + CPUID + MOVL AX, eax+8(FP) + MOVL BX, ebx+12(FP) + MOVL CX, ecx+16(FP) + MOVL DX, edx+20(FP) + RET diff --git a/source/rdt/rdt_stub.go b/source/rdt/rdt_stub.go new file mode 100644 index 000000000..4b82ca25c --- /dev/null +++ b/source/rdt/rdt_stub.go @@ -0,0 +1,23 @@ +// +build !amd64 + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rdt + +func discoverRDT() []string { + return []string{} +}