diff --git a/.gitignore b/.gitignore index 41d7a72322..883a7fa73b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ vendor pkg/client pkg/apis/policy/v1alpha1/zz_generated.deepcopy.go +certs +kube-policy +Gopkg.lock diff --git a/README.md b/README.md index fdea631fa3..362bfd17a3 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ If you don't want to use SSH, you just can clone repo with git, but ensure that ## Restore dependencies Navigate to kube-policy project dir and execute: -`dep restore` +`dep ensure` This will install necessary dependencies described in README.md # Contributing diff --git a/controller/controller.go b/controller/controller.go index 7f2d0aa9aa..15e86e601a 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -2,36 +2,36 @@ package controller import ( "time" - "fmt" + "log" - "k8s.io/sample-controller/pkg/signals" "k8s.io/client-go/tools/clientcmd" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" - clientset "nirmata/kube-policy/pkg/client/clientset/versioned" - informers "nirmata/kube-policy/pkg/client/informers/externalversions" - lister "nirmata/kube-policy/pkg/client/listers/policy/v1alpha1" - types "nirmata/kube-policy/pkg/apis/policy/v1alpha1" + clientset "github.com/nirmata/kube-policy/pkg/client/clientset/versioned" + informers "github.com/nirmata/kube-policy/pkg/client/informers/externalversions" + lister "github.com/nirmata/kube-policy/pkg/client/listers/policy/v1alpha1" + types "github.com/nirmata/kube-policy/pkg/apis/policy/v1alpha1" ) // Controller for CRD type Controller struct { policyInformerFactory informers.SharedInformerFactory policyLister lister.PolicyLister + logger *log.Logger } // NewController from cmd args -func NewController(masterURL, kubeconfigPath string) (*Controller, error) { +func NewController(masterURL, kubeconfigPath string, logger *log.Logger) (*Controller, error) { cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfigPath) if err != nil { - fmt.Printf("Error building kubeconfig: %v\n", err) + logger.Printf("Error building kubeconfig: %v\n", err) return nil, err } policyClientset, err := clientset.NewForConfig(cfg) if err != nil { - fmt.Printf("Error building policy clientset: %v\n", err) + logger.Printf("Error building policy clientset: %v\n", err) return nil, err } @@ -53,15 +53,8 @@ func NewController(masterURL, kubeconfigPath string) (*Controller, error) { } // Run is main controller thread -func (c *Controller) Run() error { - stopCh := signals.SetupSignalHandler() +func (c *Controller) Run(stopCh <-chan struct{}) { c.policyInformerFactory.Start(stopCh) - - fmt.Println("Running controller...") - <-stopCh - fmt.Println("\nShutting down controller...") - - return nil } // GetPolicies retrieves all policy resources @@ -85,24 +78,24 @@ func (c *Controller) GetPolicies() ([]*types.Policy, error) { func (c *Controller) createPolicyHandler(resource interface{}) { key := c.getResourceKey(resource) - fmt.Printf("Created policy: %s\n", key) + c.logger.Printf("Created policy: %s\n", key) } func (c *Controller) updatePolicyHandler(oldResource, newResource interface{}) { oldKey := c.getResourceKey(oldResource) newKey := c.getResourceKey(newResource) - fmt.Printf("Updated policy from %s to %s\n", oldKey, newKey) + c.logger.Printf("Updated policy from %s to %s\n", oldKey, newKey) } func (c *Controller) deletePolicyHandler(resource interface{}) { key := c.getResourceKey(resource) - fmt.Printf("Deleted policy: %s\n", key) + c.logger.Printf("Deleted policy: %s\n", key) } func (c *Controller) getResourceKey(resource interface{}) string { if key, err := cache.MetaNamespaceKeyFunc(resource); err != nil { - fmt.Printf("Error retrieving policy key: %v\n", err) + c.logger.Printf("Error retrieving policy key: %v\n", err) return "" } else { return key diff --git a/crd/MutatingWebhookConfiguration.yaml b/crd/MutatingWebhookConfiguration.yaml new file mode 100644 index 0000000000..c4466634f6 --- /dev/null +++ b/crd/MutatingWebhookConfiguration.yaml @@ -0,0 +1,19 @@ +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingWebhookConfiguration +metadata: + name: nirmata-kube-policy-webhook-cfg + labels: + app: kube-policy +webhooks: + - name: webhook.nirmata.kube-policy + clientConfig: + service: + name: kube-policy-svc + namespace: default + path: "/mutate" + caBundle: MIIC5zCCAc+gAwIBAgIBATANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwptaW5pa3ViZUNBMB4XDTE5MDIwMzE1MjM0M1oXDTI5MDIwMTE1MjM0M1owFTETMBEGA1UEAxMKbWluaWt1YmVDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOW3JNJEhX6syO6a+Vr8fezQUmHgJ+oUwYZbwIcb1TQKAGVoIPcN5nkBw11P6syjnrxoPt9HVq3/0mWJOacBgVtuAAZ4sQ8QevFwKmipTpTAC+SEBVhsypqO/1aLs2imbHQr2AVlCy2LxppX7lupl5ELwt9t5nSI3zuauezZ6ujkOCWcO52dGA3dIEXBiKiSQ4Svlqfnjpt7w8Frf6z77nmZSCbAXOf8jjPlObQGTFqzKq+gOmK3LzpANoY6VJSAjQP0jTTc7qC9u3KG53lbTectcBrcQnHRukUvfExI1YyYBTjekjN3DzTkjsn8FCar8hkR8/G4OnwZmiHgDVClrtsCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgKkMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAPvhLvSaYjT32cqy8tR7KR6PyO3lWKt1Tg1R6IrzavSp+5q9AkQyJQmgsm6WcOsxVwFHDMb23iPzv5UDQPmhmJlGRtFgHbCXOYL+Gf/f/6atez5EzbX3T/tSZPF7ASGLSClEGtOwFUYcXqOeQtInPVPe26PbG5k+XCdqDL8HvrRvyKf5HkTt/5nMYMig5TBs6L1O+GGfvM8dTNwW8w3T0ZUMoF4CKVmhMynG47hWW1HGdvqj/NWp8VWqO6Mo+6pBGJrrMdb7IArN725jhZps2CaD1bpGYVIB4Ad65E6ZbSXl12xUq+RI/LfqIaRAALJkXK3v0bfiJ1+SPMWok0QxjJ + rules: + - operations: [ "CREATE" ] + resources: [ "*/*" ] + apiGroups: [ "*" ] + apiVersions: [ "*" ] diff --git a/crd/MutatingWebhookConfiguration_local.yaml b/crd/MutatingWebhookConfiguration_local.yaml new file mode 100644 index 0000000000..38415c7e4f --- /dev/null +++ b/crd/MutatingWebhookConfiguration_local.yaml @@ -0,0 +1,15 @@ +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingWebhookConfiguration +metadata: + name: nirmata-kube-policy-webhook-cfg + labels: + app: kube-policy +webhooks: + - name: webhook.nirmata.kube-policy + clientConfig: + url: "https://localhost/mutate" + rules: + - operations: [ "CREATE" ] + resources: [ "*/*" ] + apiGroups: [ "*" ] + apiVersions: [ "*" ] diff --git a/crd/crd.yaml b/crd/crd.yaml index b144a19328..d4028c8be8 100644 --- a/crd/crd.yaml +++ b/crd/crd.yaml @@ -8,9 +8,8 @@ spec: - name: v1alpha1 served: true storage: true - scope: Namespaced + scope: Cluster names: kind: Policy plural: policies singular: policy - version: v1alpha1 diff --git a/crd/deployment.yaml b/crd/deployment.yaml new file mode 100644 index 0000000000..907ac8d8e4 --- /dev/null +++ b/crd/deployment.yaml @@ -0,0 +1,28 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: kube-policy-deployment + labels: + app: kube-policy +spec: + replicas: 1 + template: + metadata: + labels: + app: kube-policy + spec: + containers: + - name: kube-policy + image: nirmata/kube-policy:latest + imagePullPolicy: IfNotPresent + args: + - -cert=/etc/kube-policy/certs/server.crt + - -key=/etc/kube-policy/certs/server-key.pem + volumeMounts: + - name: kube-policy-certs + mountPath: /etc/kube-policy/certs + readOnly: true + volumes: + - name: kube-policy-certs + secret: + secretName: kube-policy-secret diff --git a/crd/service.yaml b/crd/service.yaml new file mode 100644 index 0000000000..e42b03f03b --- /dev/null +++ b/crd/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: kube-policy-svc + labels: + app: kube-policy +spec: + ports: + - port: 443 + targetPort: 443 + selector: + app: kube-policy diff --git a/main.go b/main.go index e36e7a24e4..4ab71a500b 100644 --- a/main.go +++ b/main.go @@ -1,33 +1,59 @@ package main import ( + "log" + "os" "flag" "fmt" - controller "nirmata/kube-policy/controller" + "github.com/nirmata/kube-policy/controller" + "github.com/nirmata/kube-policy/server" + + "k8s.io/sample-controller/pkg/signals" ) var ( masterURL string kubeconfig string + cert string + key string ) func main() { - flag.Parse() + flag.Parse() - controller, err := controller.NewController(masterURL, kubeconfig) + if cert == "" || key == "" { + log.Fatal("TLS certificate or/and key is not set") + } + + httpLogger := log.New(os.Stdout, "http: ", log.LstdFlags|log.Lshortfile) + crdcLogger := log.New(os.Stdout, "crdc: ", log.LstdFlags|log.Lshortfile) + + server := server.NewWebhookServer(cert, key, httpLogger) + server.RunAsync() + + controller, err := controller.NewController(masterURL, kubeconfig, crdcLogger) if err != nil { fmt.Printf("Error creating Controller! Error: %s\n", err) return } - err = controller.Run() + stopCh := signals.SetupSignalHandler() + controller.Run(stopCh) + if err != nil { fmt.Printf("Error running Controller! Error: %s\n", err) } + + fmt.Printf("Policy Controller has started") + <-stopCh + server.Stop() + fmt.Printf("Policy Controller has stopped") } func init() { flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") flag.StringVar(&masterURL, "master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.") -} + flag.StringVar(&cert, "cert", "", "TLS certificate used in connection with cluster.") + flag.StringVar(&key, "key", "", "Key, used in TLS connection.") +} \ No newline at end of file diff --git a/pkg/apis/policy/v1alpha1/register.go b/pkg/apis/policy/v1alpha1/register.go index d3d78196e9..dd21eacbfc 100644 --- a/pkg/apis/policy/v1alpha1/register.go +++ b/pkg/apis/policy/v1alpha1/register.go @@ -5,7 +5,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "nirmata/kube-policy/pkg/apis/policy" + "github.com/nirmata/kube-policy/pkg/apis/policy" ) // SchemeGroupVersion is group version used to register these objects diff --git a/pkg/apis/policy/v1alpha1/types.go b/pkg/apis/policy/v1alpha1/types.go index 8d7ce42ca5..a3db0ce9b2 100644 --- a/pkg/apis/policy/v1alpha1/types.go +++ b/pkg/apis/policy/v1alpha1/types.go @@ -26,7 +26,7 @@ type PolicySpec struct { type PolicyRule struct { Resource PolicyResource `json:"resource"` Patches []PolicyPatch `json:"patches"` - Generator *PolicyConfigGenerator `json:"generator"` + Generators []PolicyConfigGenerator `json:"generator"` } // PolicyResource describes the resource rule applied to @@ -47,7 +47,7 @@ type PolicyPatch struct { type PolicyConfigGenerator struct { Name string `json:"name"` CopyFrom *PolicyCopyFrom `json:"copyFrom"` - Data *map[string]string `json:"data"` + Data map[string]string `json:"data"` } // PolicyCopyFrom is TODO diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000000..f336e95ec5 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,25 @@ +Use these scripts to prepare the controller for work. +All these scripts should be launched from the root folder of the project, for example: +`scripts/compile-image.sh` + +### compile-image.sh ### +Compiles the project to go executable, generates docker image and pushes it to the repo. Has no arguments. + +### generate-server-cert.sh ### +Generates TLS certificate and key that used by webhook server. Example: +`scripts/generate-server-cert.sh --service=kube-policy-svc --namespace=my_namespace --serverIp=192.168.10.117` +* `--service` identifies the service for in-cluster webhook server. Do not specify it if you plan to run webhook server outside the cluster. +* `--namespace` identifies the namespace for in-cluster webhook server. Default value is "default". +* `--serverIp` is the IP of master node, it can be found in `~/.kube/config`: clusters.cluster[0].server. **The default is hardcoded value**, so you should explicitly specify it. + +### deploy-controller.sh ### +Prepares controller for current environment in 1 of 2 possible modes: free (local) and in-cluster. Usage: +`scripts/deploy-controller.sh --namespace=my_namespace --serverIp=192.168.10.117` +* --namespace identifies the namespace for in-cluster webhook server. Do not specify it if you plan to run webhook server outside the cluster. +* --serverIp is the IP of master node, means the same as for `generate-server-cert.sh`. + +### test-web-hook.sh ### +Quickly creates and deletes test config map. If your webhook server is running, you should see the corresponding output from it. Use this script after `deploy-controller.sh`. + +### update-codegen.sh ### +Generates additional code for controller object. You should resolve all dependencies before using it, see main Readme for details. diff --git a/scripts/compile-image.sh b/scripts/compile-image.sh new file mode 100755 index 0000000000..19466975b0 --- /dev/null +++ b/scripts/compile-image.sh @@ -0,0 +1,23 @@ +#!/bin/bash +hub_user_name="nirmata" +project_name="kube-policy" +version="latest" + +echo "# Ensuring Go dependencies..." +#dep ensure || exit 2 + +echo "# Building executable ${project_name}..." +CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o ${project_name} . || exit 3 + +echo "# Building docker image ${hub_user_name}/${project_name}:${version}" +cat < Dockerfile +FROM alpine:latest +WORKDIR ~/ +ADD ${project_name} ./${project_name} +ENTRYPOINT ["./${project_name}"] +EOF +tag="${hub_user_name}/${project_name}:${version}" +docker build --no-cache -t "${tag}" . || exit 4 + +echo "# Pushing image to repository..." +docker push "${tag}" || exit 5 diff --git a/scripts/deploy-controller.sh b/scripts/deploy-controller.sh new file mode 100755 index 0000000000..50517137bb --- /dev/null +++ b/scripts/deploy-controller.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +for i in "$@" +do +case $i in + --namespace=*) + namespace="${i#*=}" + shift + ;; + --serverIp=*) + serverIp="${i#*=}" + shift + ;; +esac +done + +if [ -z "${serverIp}" ]; then + # This is the standard IP of minikube + serverIp="192.168.10.117" #TODO: ! Read it from ~/.kube/config ! +fi + +hub_user_name="nirmata" +project_name="kube-policy" + +service_name="${project_name}-svc" +echo "Generating certificate for the service ${service_name}..." + +certsGenerator="./scripts/generate-server-cert.sh" +chmod +x "${certsGenerator}" + +if [ -z "${namespace}" ]; then # controller is launched locally + + ${certsGenerator} "--serverIp=${serverIp}" || exit 2 + + echo "Applying webhook..." + kubectl delete -f crd/MutatingWebhookConfiguration_local.yaml + kubectl create -f crd/MutatingWebhookConfiguration_local.yaml || exit 3 + + echo -e "\n### You can build and run kube-policy project locally.\n### To check its work, run it with parameters -cert and -key, which contain generated TLS certificate and key (see their paths in log above)." + +else # controller is launched within a cluster + + ${certsGenerator} "--service=${service_name}" "--namespace=${namespace}" "--serverIp=${serverIp}" || exit 2 + + secret_name="${project_name}-secret" + echo "Generating secret ${secret_name}..." + kubectl delete secret "${secret_name}" 2>/dev/null + kubectl create secret generic ${secret_name} --namespace ${namespace} --from-file=./certs || exit 3 + + echo "Creating the service ${service_name}..." + kubectl delete -f crd/service.yaml + kubectl create -f crd/service.yaml || exit 4 + + echo "Creating deployment..." + kubectl delete -f crd/deployment.yaml + kubectl create -f crd/deployment.yaml || exit 5 + + echo "Applying webhook..." + kubectl delete -f crd/MutatingWebhookConfiguration.yaml + kubectl create -f crd/MutatingWebhookConfiguration.yaml || exit 3 + + echo -e "\n### Controller is running in cluster.\n### You can use compile-image.sh to rebuild its image and then the current script to redeploy the controller.\n### Check its work by 'kubectl logs command'" + +fi diff --git a/scripts/generate-server-cert.sh b/scripts/generate-server-cert.sh new file mode 100755 index 0000000000..2dee26f47a --- /dev/null +++ b/scripts/generate-server-cert.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +for i in "$@" +do +case $i in + --service=*) + service="${i#*=}" + shift + ;; + --namespace=*) + namespace="${i#*=}" + shift + ;; + --serverIp=*) + serverIp="${i#*=}" + shift + ;; +esac +done + +if [ -z "${namespace}" ]; then + namespace="default" +fi + +echo "service is $service" +echo "namespace is $namespace" +echo "serverIp is $serverIp" + +destdir="certs" +if [ ! -d "$destdir" ]; then + mkdir ${destdir} || exit 1 +fi +tmpdir=$(mktemp -d) + +cat <> ${tmpdir}/csr.conf +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = @alt_names +[alt_names] +DNS.1 = ${service} +DNS.2 = ${service}.${namespace} +DNS.3 = ${service}.${namespace}.svc +DNS.4 = ${serverIp} +EOF + +outKeyFile=${destdir}/server-key.pem +outCertFile=${destdir}/server.crt + +openssl genrsa -out ${outKeyFile} 2048 || exit 2 +if [ ! -z "${service}" ]; then + subjectCN="${service}.${namespace}.svc" + echo "Configuring work WITHIN a cluster with CN=${subjectCN}" +else + subjectCN=${serverIp} + echo "Configuring work OUTSIDE a cluster with CN=${subjectCN}" +fi + +openssl req -new -key ${destdir}/server-key.pem -subj "/CN=${subjectCN}" -out ${tmpdir}/server.csr -config ${tmpdir}/csr.conf || exit 3 + +CSR_NAME=${service}.cert-request +kubectl delete csr ${CSR_NAME} 2>/dev/null + +cat < ${outCertFile} || exit 5 + +echo "Generated:" +echo ${outKeyFile} +echo ${outCertFile} diff --git a/scripts/test-web-hook.sh b/scripts/test-web-hook.sh new file mode 100755 index 0000000000..94d9f061b7 --- /dev/null +++ b/scripts/test-web-hook.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# You should see the trace of requests in the output of webhook server +kubectl create configmap test-config-map --from-literal="some_var=some_value" +kubectl delete configmap test-config-map diff --git a/server/server.go b/server/server.go new file mode 100644 index 0000000000..a8d8f89a89 --- /dev/null +++ b/server/server.go @@ -0,0 +1,76 @@ +package server + +import ( + "net/http/httputil" + "net/http" + "crypto/tls" + "context" + "time" + "log" + "os" +) + +// WebhookServer is a struct that describes +// TLS server with mutation webhook +type WebhookServer struct { + server http.Server + logger *log.Logger +} + +func (ws *WebhookServer) serve(w http.ResponseWriter, r *http.Request) { + dump, _ := httputil.DumpRequest(r, true) + ws.logger.Printf("%s", dump) +} + +// RunAsync runs TLS server in separate +// thread and returns control immediately +func (ws *WebhookServer) RunAsync() { + go func(ws *WebhookServer) { + err := ws.server.ListenAndServeTLS("", "") + if err != nil { + ws.logger.Fatal(err) + } + }(ws) +} + +// Stop stops TLS server +func (ws *WebhookServer) Stop() { + err := ws.server.Shutdown(context.Background()) + if err != nil { + // Error from closing listeners, or context timeout: + ws.logger.Printf("Server Shutdown error: %v", err) + ws.server.Close() + } +} + +// NewWebhookServer creates new instance of WebhookServer and configures it +func NewWebhookServer(certFile string, keyFile string, logger *log.Logger) *WebhookServer { + if logger == nil { + logger = log.New(os.Stdout, "", log.LstdFlags|log.Lshortfile) + } + + var config tls.Config + pair, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + logger.Fatal("Unable to load certificate and key: ", err) + } + config.Certificates = []tls.Certificate{pair} + + mux := http.NewServeMux() + + ws := &WebhookServer { + server: http.Server { + Addr: ":443", // Listen on port for HTTPS requests + TLSConfig: &config, + Handler: mux, + ErrorLog: logger, + ReadTimeout: 5 * time.Second, + WriteTimeout: 5 * time.Second, + }, + logger: logger, + } + + mux.HandleFunc("/mutate", ws.serve) + + return ws +}