From d96f50e7c627fefc8fa57548cc4756cf8830de3e Mon Sep 17 00:00:00 2001 From: Balaji Subramaniam Date: Tue, 12 Jul 2016 02:30:49 +0200 Subject: [PATCH] 1. Enabled RDT discovery. 2. Updated the ds to a job for node labelling. 3. Added script to run a job in each node. 4. Updated Readme. --- Dockerfile | 32 +++- LICENSE | 177 ++++++++++++++++++ README.md | 92 ++++++++- ....json => featurelabeling-job.json.template | 17 +- label-nodes.sh | 9 + main.go | 56 +++++- rdt-discovery/Makefile | 16 ++ rdt-discovery/l2-allocation-discovery.c | 40 ++++ rdt-discovery/l3-allocation-discovery.c | 43 +++++ rdt-discovery/monitoring-discovery.c | 40 ++++ 10 files changed, 511 insertions(+), 11 deletions(-) create mode 100644 LICENSE rename featurelabeling-daemonset.json => featurelabeling-job.json.template (80%) create mode 100755 label-nodes.sh create mode 100644 rdt-discovery/Makefile create mode 100644 rdt-discovery/l2-allocation-discovery.c create mode 100644 rdt-discovery/l3-allocation-discovery.c create mode 100644 rdt-discovery/monitoring-discovery.c diff --git a/Dockerfile b/Dockerfile index 464c7fd32..307804063 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1 +1,31 @@ -FROM golang:onbuild +#======================================================================== +# Copyright 2016 Intel Corporation +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#======================================================================== + +FROM golang:latest + +ADD ./main.go /go/src/github.com/intelsdi-x/dbi-iafeature-discovery/ +ADD ./rdt-discovery/mon-discovery /go/bin/ +ADD ./rdt-discovery/l3-alloc-discovery /go/bin/ +ADD ./rdt-discovery/l2-alloc-discovery /go/bin/ + +WORKDIR /go/src/github.com/intelsdi-x/dbi-iafeature-discovery + +RUN git clone https://github.com/01org/intel-cmt-cat.git +RUN cd intel-cmt-cat/lib; make install + +RUN go get -d -v github.com/klauspost/cpuid +RUN go get -d -v k8s.io/kubernetes; exit 0 + +RUN go install github.com/intelsdi-x/dbi-iafeature-discovery + +ENTRYPOINT /go/bin/dbi-iafeature-discovery diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..f433b1a53 --- /dev/null +++ b/LICENSE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/README.md b/README.md index 5356e9d86..4dfef8055 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,90 @@ -# dbi-iafeature-discovery -Research Prototype for IA feature discovery in a Kubernetes cluster. Details on this prototype can be found here: https://goo.gl/e5xe5Q. +#CPU Feature Discovery for Kubernetes +This software enables Intel Architecture (IA) feature discovery for Kubernetes. It detects the CPU features, such as Intel [Resource Director Technology][intel-rdt] (RDT), available in each node of the Kubernetes cluster and labels them appropriately. + + - [Getting Started](#getting-started) + * [System Requirements](#system-requirements) + * [Installation](#installation) + * [Usage](#usage) + - [Documentation](#documentation) + - [License](#license) + +##Getting Started +### System Requirements +At a minimum, you will need: + 1. [Docker] [docker-down] (only required to build and push docker images) + 2. [GCC] [gcc-down] (only required to build software to detect Intel RDT feature set) + 3. Linux (x86_64) + 4. [kubectl] [kubectl-setup] (properly setup and configured to work with a Kubernetes cluster) + +###Installation +####Downloading the Source Code +``` +git clone https://github.com/intelsdi-x/dbi-iafeature-discovery +``` + +####Building from Source +The build steps described below are optional. The default docker image in Docker hub at `intelsdi/nodelabels` can be used to decorate the Kubernetes node with IA features. Skip to usage instructions if you do not need to build your own docker image. + +Build the Intel RDT Detection Software Using `make` (Optional): +``` +cd /rdt-discovery +make +``` + +Build the Docker Image (Optional): +``` +cd +docker build -t / . +``` + +Push the Docker Image (Optional) +``` +docker push +``` + +Change the Job Spec to Use the Appropriate Image (Optional): +Change line #40 in `featurelabeling-job.json.template` to the appropriate image name in the `/` format used for building the image above or use the default `intelsdi/nodelabels` image. + +###Usage +Deploying Kubernetes Job for Node Feature Labeling: +``` +./label-nodes.sh +``` +The above command will label each node in the cluster with the features from cpuid. The labeling format used is +``` + = <"node.alpha.intel.com/", "true"> +``` +Note that only features that are available in that node are labelled. + +##Documentation +This software determines the Intel Architecture (IA) features associated with a processor using `cpuid`. The following set of features are discovered and labelled by this software. + +###Intel Resource Director Technology (RDT) Features +| Label | Feature(s) Represented | +| :------------: | :---------------------------------------------------------------------------------: | +| RDTMON | Intel Cache Monitoring Technology (CMT) and Intel Memory Bandwidth Monitoring (MBM) | +| RDTL3CA | Intel L3 Cache Allocation Technology | +| RDTL2CA | Intel L2 Cache Allocation Technology | + +###Other Features (Partial List) +| Label | Feature(s) Represented | +| :------------: | :---------------------------------------------------------------------------------: | +| ADX | Multi-Precision Add-Carry Instruction Extensions (ADX) | +| AESNI | Advanced Encryption Standard (AES) New Instructions (AES-NI) | +| AVX | Advanced Vector Extensions (AVX)| +| AVX2 | Advanced Vector Extensions 2 (AVX2)| +| BMI1 | Bit Manipulation Instruction Set 1 (BMI)| +| BMI2 | Bit Manipulation Instruction Set 2 (BMI2)| +| SSE4.1 | Streaming SIMD Extensions 4.1 (SSE4.1)| +| SSE4.2 | Streaming SIMD Extensions 4.2 (SSE4.2)| +| SGX | Software Guard Extensions (SGX) | + +##License +This is an Open Source software released under the Apache 2.0 [License](LICENSE). + +[intel-rdt]: http://www.intel.com/content/www/us/en/architecture-and-technology/resource-director-technology.html +[docker-down]: https://docs.docker.com/engine/installation/ +[golang-down]: https://golang.org/dl/ +[gcc-down]: https://gcc.gnu.org/ +[kubectl-setup]: https://coreos.com/kubernetes/docs/latest/configure-kubectl.html +[balaji-github]: https://github.com/balajismaniam diff --git a/featurelabeling-daemonset.json b/featurelabeling-job.json.template similarity index 80% rename from featurelabeling-daemonset.json rename to featurelabeling-job.json.template index ff895b5e9..9d87e3453 100644 --- a/featurelabeling-daemonset.json +++ b/featurelabeling-job.json.template @@ -1,6 +1,6 @@ { "apiVersion": "extensions/v1beta1", - "kind": "DaemonSet", + "kind": "Job", "metadata": { "labels": { "app": "dbi-nodelabel" @@ -8,6 +8,8 @@ "name": "dbi-nodelabel" }, "spec": { + "completions": COMPLETION_COUNT, + "parallelism": PARALLELISM_COUNT, "template": { "metadata": { "labels": { @@ -37,14 +39,15 @@ ], "image": "intelsdi/nodelabels", "name": "nodelabeler", - "resources": { - "limits": { - "cpu": "1000m", - "memory": "1000Mi" + "ports": [ + { + "containerPort": 2233, + "hostPort": 7156 } - } + ] } - ] + ], + "restartPolicy": "Never" } } } diff --git a/label-nodes.sh b/label-nodes.sh new file mode 100755 index 000000000..9f004d284 --- /dev/null +++ b/label-nodes.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +NumNodes=$(kubectl get nodes | grep -i ready | wc -l) +sed -e "s/COMPLETION_COUNT/$NumNodes/" -e "s/PARALLELISM_COUNT/$NumNodes/" featurelabeling-job.json.template > featurelabeling-job.json + +kubectl create -f featurelabeling-job.json + + + diff --git a/main.go b/main.go index 612a90b0e..b7f48b2cc 100644 --- a/main.go +++ b/main.go @@ -1,8 +1,22 @@ +/* +Copyright 2016 Intel Corporation +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package main import ( "log" "os" + "os/exec" "github.com/klauspost/cpuid" client "k8s.io/kubernetes/pkg/client/unversioned" @@ -15,25 +29,65 @@ func main() { log.Fatalf("Can't Get K8s Client:%v", err) } + //Get the cpu features as strings features := cpuid.CPU.Features.Strings() + //Get the pod name and namespace from the env variables podName := os.Getenv("POD_NAME") podns := os.Getenv("POD_NAMESPACE") + //Get the pod object using the pod name and namespace pod, err := cli.Pods(podns).Get(podName) if err != nil { log.Fatalf("Can't Get Pod:%v", err) } + //Get the node object using the pod name and namespace node, err := cli.Nodes().Get(pod.Spec.NodeName) if err != nil { log.Fatalf("Can't Get Node:%v", err) } + //Add each of the cpu feature as the node label for _, feature := range features { - node.Labels["node.alpha.intel.com/"+feature] = "true" + node.Labels["scheduler.alpha.intel.com/"+feature] = "true" } + //If supported, add CMT, MBM and CAT features as a node label + cmd := "/go/bin/mon-discovery" + out, err := exec.Command("bash", "-c", cmd).Output() + if err != nil { + log.Fatalf("Can't Dectect Support for RDT Monitoring:%v", err) + } + + outString := string(out[:]) + if outString == "DETECTED" { + node.Labels["scheduler.alpha.intel.com/RDTMON"] = "true" + } + + cmd = "/go/bin/l3-alloc-discovery" + out, err = exec.Command("bash", "-c", cmd).Output() + if err != nil { + log.Fatalf("Can't Dectect Support for RDT L3 Allocation:%v", err) + } + + outString = string(out[:]) + if outString == "DETECTED" { + node.Labels["scheduler.alpha.intel.com/RDTL3CA"] = "true" + } + + cmd = "/go/bin/l2-alloc-discovery" + out, err = exec.Command("bash", "-c", cmd).Output() + if err != nil { + log.Fatalf("Can't Dectect Support for RDT L2 Allocation:%v", err) + } + + outString = string(out[:]) + if outString == "DETECTED" { + node.Labels["scheduler.alpha.intel.com/RDTL2CA"] = "true" + } + + //Update the node with the node labels _, err = cli.Nodes().Update(node) if err != nil { log.Fatalf("Can't Update Node:%v", err) diff --git a/rdt-discovery/Makefile b/rdt-discovery/Makefile new file mode 100644 index 000000000..b979fa77c --- /dev/null +++ b/rdt-discovery/Makefile @@ -0,0 +1,16 @@ +CC=gcc +LIBDIR=/usr/local/lib +INCDIR=$(HOME)/intel-cmt-cat/lib/ + +LDFLAGS=-L$(LIBDIR) +LDLIBS=-lpqos + +CFLAGS=-I$(INCDIR) + + +default: + $(MAKE) all +all: + $(CC) $(CFLAGS) -o mon-discovery monitoring-discovery.c $(LDFLAGS) $(LDLIBS) + $(CC) $(CFLAGS) -o l3-alloc-discovery l3-allocation-discovery.c $(LDFLAGS) $(LDLIBS) + $(CC) $(CFLAGS) -o l2-alloc-discovery l2-allocation-discovery.c $(LDFLAGS) $(LDLIBS) diff --git a/rdt-discovery/l2-allocation-discovery.c b/rdt-discovery/l2-allocation-discovery.c new file mode 100644 index 000000000..f02923c68 --- /dev/null +++ b/rdt-discovery/l2-allocation-discovery.c @@ -0,0 +1,40 @@ +/* +Copyright 2016 Intel Corporation +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include +#include +#include "machine.h" + +int main(int argc, char *argv[]) { + int ret, det=1; + struct cpuid_out res; + + //Logic below from https://github.com/01org/intel-cmt-cat/blob/master/lib/host_cap.c + lcpuid(0x7, 0x0, &res); + if (!(res.ebx & (1 << 15))) { + det = 0; + printf("NOT DETECTED"); + } + else { + lcpuid(0x10, 0x0, &res); + if (!(res.ebx & (1 << 2))) { + det = 0; + printf("NOT DETECTED"); + } + } + + if (det) + printf("DETECTED"); + + return 0; +} diff --git a/rdt-discovery/l3-allocation-discovery.c b/rdt-discovery/l3-allocation-discovery.c new file mode 100644 index 000000000..c4a3219cf --- /dev/null +++ b/rdt-discovery/l3-allocation-discovery.c @@ -0,0 +1,43 @@ +/* +Copyright 2016 Intel Corporation +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include +#include +#include "machine.h" + +int main(int argc, char *argv[]) { + int ret, det=1; + struct cpuid_out res; + + //Logic below from https://github.com/01org/intel-cmt-cat/blob/master/lib/host_cap.c + //TODO(balajismaniam): Implement L3 CAT detection using brand string and MSR probing if + //not detected using cpuid + + lcpuid(0x7, 0x0, &res); + if (!(res.ebx & (1 << 15))) { + det = 0; + printf("NOT DETECTED"); + } + else { + lcpuid(0x10, 0x0, &res); + if (!(res.ebx & (1 << 1))) { + det = 0; + printf("NOT DETECTED"); + } + } + + if (det) + printf("DETECTED"); + + return 0; +} diff --git a/rdt-discovery/monitoring-discovery.c b/rdt-discovery/monitoring-discovery.c new file mode 100644 index 000000000..14b4cae5c --- /dev/null +++ b/rdt-discovery/monitoring-discovery.c @@ -0,0 +1,40 @@ +/* +Copyright 2016 Intel Corporation +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include +#include +#include "machine.h" + +int main(int argc, char *argv[]) { + int ret, det=1; + struct cpuid_out res; + + //Logic below from https://github.com/01org/intel-cmt-cat/blob/master/lib/host_cap.c + lcpuid(0x7, 0x0, &res); + if (!(res.ebx & (1 << 12))) { + det = 0; + printf("NOT DETECTED"); + } + else { + lcpuid(0xf, 0x0, &res); + if (!(res.edx & (1 << 1))) { + det=0; + printf("NOT DETECTED"); + } + } + + if (det) + printf("DETECTED"); + + return 0; +}