Allow user to run Kyverno in debug mode

2024-12-14 11:57:48 +00:00 · 2019-06-10 18:10:51 -07:00 · 2019-06-10 18:10:51 -07:00 · 1013a8a637
commit 1013a8a637
parent d0e1524480
7 changed files with 336 additions and 9 deletions
--- a/definitions/install_debug.yaml
+++ b/definitions/install_debug.yaml
@ -0,0 +1,126 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: policies.kyverno.io
+spec:
+  group: kyverno.io
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+  scope: Cluster
+  names:
+    kind: Policy
+    plural: policies
+    singular: policy
+  subresources:
+    status: {}
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          required:
+          - rules
+          properties:
+            rules:
+              type: array
+              items:
+                type: object
+                required:
+                - name
+                - resource
+                properties:
+                  name:
+                    type: string
+                  resource:
+                    type: object
+                    required:
+                    - kinds
+                    properties:
+                      kinds:
+                        type: array
+                        items:
+                          type: string
+                      name:
+                        type: string
+                      selector:
+                        properties:
+                          matchLabels:
+                            type: object
+                            additionalProperties:
+                              type: string
+                          matchExpressions:
+                            type: array
+                            items:
+                              type: object
+                              required:
+                              - key
+                              - operator
+                              properties:
+                                key:
+                                  type: string
+                                operator:
+                                  type: string
+                                values:
+                                  type: array
+                                  items:
+                                    type: string
+                  mutate:
+                    type: object
+                    properties:
+                      overlay:
+                        AnyValue: {}
+                      patches:
+                        type: array
+                        items:
+                          type: object
+                          required:
+                          - path
+                          - op
+                          properties:
+                            path:
+                              type: string
+                            op:
+                              type: string
+                              enum:
+                              - add
+                              - replace
+                              - remove
+                            value:
+                              AnyValue: {}
+                  validate:
+                    type: object
+                    required:
+                    - pattern
+                    properties:
+                      message:
+                        type: string
+                      pattern:
+                        AnyValue: {}
+                  generate:
+                    type: object
+                    required:
+                    - kind
+                    - name
+                    properties:
+                      kind:
+                        type: string
+                      name:
+                        type: string
+                      clone: 
+                        type: object
+                        required:
+                        - namespace
+                        - name
+                        properties:
+                          namespace:
+                            type: string
+                          name:
+                            type: string
+                      data:
+                        AnyValue: {}
+---
+kind: Namespace
+apiVersion: v1
+metadata: 
+    name: "kyverno"
--- a/init.go
+++ b/init.go
@ -1,6 +1,8 @@
 package main

 import (
+	"fmt"
+
 	"github.com/golang/glog"
 	client "github.com/nirmata/kyverno/pkg/dclient"
 	tls "github.com/nirmata/kyverno/pkg/tls"
@ -40,10 +42,12 @@ func initTLSPemPair(configuration *rest.Config, client *client.Client) (*tls.Tls
 		if err != nil {
 			return nil, err
 		}
-		err = client.WriteTlsPair(certProps, tlsPair)
-		if err != nil {
-			glog.Errorf("Unable to save TLS pair to the cluster: %v", err)
+		if err = client.WriteTlsPair(certProps, tlsPair); err != nil {
+			return nil, fmt.Errorf("Unable to save TLS pair to the cluster: %v", err)
 		}
+		return tlsPair, nil
 	}
+
+	glog.Infoln("Using existing TLS key/certificate pair")
 	return tlsPair, nil
 }
--- a/main.go
+++ b/main.go
@ -16,6 +16,7 @@ import (

 var (
 	kubeconfig string
+	serverIP   string
 )

 func main() {
@ -55,7 +56,7 @@ func main() {
 		glog.Fatalf("Unable to create webhook server: %v\n", err)
 	}

-	webhookRegistrationClient, err := webhooks.NewWebhookRegistrationClient(clientConfig, client)
+	webhookRegistrationClient, err := webhooks.NewWebhookRegistrationClient(clientConfig, client, serverIP)
 	if err != nil {
 		glog.Fatalf("Unable to register admission webhooks on cluster: %v\n", err)
 	}
@ -81,6 +82,7 @@ func main() {

 func init() {
 	flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
+	flag.StringVar(&serverIP, "serverIP", "", "IP address where Kyverno controller runs. Only required if out-of-cluster.")
 	config.LogDefaultFlags()
 	flag.Parse()
 }
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@ -7,11 +7,13 @@ const (
 	KubePolicyNamespace = "kyverno"
 	WebhookServiceName  = "kyverno-svc"

-	MutatingWebhookConfigurationName = "kyverno-mutating-webhook-cfg"
-	MutatingWebhookName              = "nirmata.kyverno.mutating-webhook"
+	MutatingWebhookConfigurationName  = "kyverno-mutating-webhook-cfg"
+	MutatingWebhookConfigurationDebug = "kyverno-mutating-webhook-cfg-debug"
+	MutatingWebhookName               = "nirmata.kyverno.mutating-webhook"

-	ValidatingWebhookConfigurationName = "kyverno-validating-webhook-cfg"
-	ValidatingWebhookName              = "nirmata.kyverno.validating-webhook"
+	ValidatingWebhookConfigurationName  = "kyverno-validating-webhook-cfg"
+	ValidatingWebhookConfigurationDebug = "kyverno-validating-webhook-cfg-debug"
+	ValidatingWebhookName               = "nirmata.kyverno.validating-webhook"

 	// Due to kubernetes issue, we must use next literal constants instead of deployment TypeMeta fields
 	// Issue: https://github.com/kubernetes/kubernetes/pull/63972
--- a/pkg/webhooks/registration.go
+++ b/pkg/webhooks/registration.go
@ -2,8 +2,10 @@ package webhooks

 import (
 	"errors"
+	"fmt"
 	"io/ioutil"

+	"github.com/golang/glog"
 	"github.com/nirmata/kyverno/pkg/config"
 	client "github.com/nirmata/kyverno/pkg/dclient"

@ -18,10 +20,12 @@ type WebhookRegistrationClient struct {
 	registrationClient *admregclient.AdmissionregistrationV1beta1Client
 	client             *client.Client
 	clientConfig       *rest.Config
+	// serverIP should be used if running Kyverno out of clutser
+	serverIP string
 }

 // NewWebhookRegistrationClient creates new WebhookRegistrationClient instance
-func NewWebhookRegistrationClient(clientConfig *rest.Config, client *client.Client) (*WebhookRegistrationClient, error) {
+func NewWebhookRegistrationClient(clientConfig *rest.Config, client *client.Client, serverIP string) (*WebhookRegistrationClient, error) {
 	registrationClient, err := admregclient.NewForConfig(clientConfig)
 	if err != nil {
 		return nil, err
@ -31,11 +35,15 @@ func NewWebhookRegistrationClient(clientConfig *rest.Config, client *client.Clie
 		registrationClient: registrationClient,
 		client:             client,
 		clientConfig:       clientConfig,
+		serverIP:           serverIP,
 	}, nil
 }

 // Register creates admission webhooks configs on cluster
 func (wrc *WebhookRegistrationClient) Register() error {
+	if wrc.serverIP != "" {
+		glog.Infof("Registering webhook with url https://%s\n", wrc.serverIP)
+	}
 	// For the case if cluster already has this configs
 	wrc.Deregister()

@ -66,6 +74,12 @@ func (wrc *WebhookRegistrationClient) Register() error {
 // This function does not fail on error:
 // Register will fail if the config exists, so there is no need to fail on error
 func (wrc *WebhookRegistrationClient) Deregister() {
+	if wrc.serverIP != "" {
+		wrc.registrationClient.MutatingWebhookConfigurations().Delete(config.MutatingWebhookConfigurationDebug, &meta.DeleteOptions{})
+		wrc.registrationClient.ValidatingWebhookConfigurations().Delete(config.ValidatingWebhookConfigurationDebug, &meta.DeleteOptions{})
+		return
+	}
+
 	wrc.registrationClient.MutatingWebhookConfigurations().Delete(config.MutatingWebhookConfigurationName, &meta.DeleteOptions{})
 	wrc.registrationClient.ValidatingWebhookConfigurations().Delete(config.ValidatingWebhookConfigurationName, &meta.DeleteOptions{})
 }
@ -83,6 +97,10 @@ func (wrc *WebhookRegistrationClient) constructMutatingWebhookConfig(configurati
 		return nil, errors.New("Unable to extract CA data from configuration")
 	}

+	if wrc.serverIP != "" {
+		return wrc.contructDebugMutatingWebhookConfig(caData), nil
+	}
+
 	return &admregapi.MutatingWebhookConfiguration{
 		ObjectMeta: meta.ObjectMeta{
 			Name:   config.MutatingWebhookConfigurationName,
@ -100,6 +118,24 @@ func (wrc *WebhookRegistrationClient) constructMutatingWebhookConfig(configurati
 	}, nil
 }

+func (wrc *WebhookRegistrationClient) contructDebugMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration {
+	url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.MutatingWebhookServicePath)
+	glog.V(3).Infof("Debug MutatingWebhookConfig is registered with url %s\n", url)
+
+	return &admregapi.MutatingWebhookConfiguration{
+		ObjectMeta: meta.ObjectMeta{
+			Name:   config.MutatingWebhookConfigurationDebug,
+			Labels: config.KubePolicyAppLabels,
+		},
+		Webhooks: []admregapi.Webhook{
+			constructDebugWebhook(
+				config.MutatingWebhookName,
+				url,
+				caData),
+		},
+	}
+}
+
 func (wrc *WebhookRegistrationClient) constructValidatingWebhookConfig(configuration *rest.Config) (*admregapi.ValidatingWebhookConfiguration, error) {
 	// Check if ca is defined in the secret tls-ca
 	// assume the key and signed cert have been defined in secret tls.kyverno
@ -112,6 +148,10 @@ func (wrc *WebhookRegistrationClient) constructValidatingWebhookConfig(configura
 		return nil, errors.New("Unable to extract CA data from configuration")
 	}

+	if wrc.serverIP != "" {
+		return wrc.contructDebugValidatingWebhookConfig(caData), nil
+	}
+
 	return &admregapi.ValidatingWebhookConfiguration{
 		ObjectMeta: meta.ObjectMeta{
 			Name:   config.ValidatingWebhookConfigurationName,
@ -129,6 +169,24 @@ func (wrc *WebhookRegistrationClient) constructValidatingWebhookConfig(configura
 	}, nil
 }

+func (wrc *WebhookRegistrationClient) contructDebugValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration {
+	url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.ValidatingWebhookServicePath)
+	glog.V(3).Infof("Debug ValidatingWebhookConfig is registered with url %s\n", url)
+
+	return &admregapi.ValidatingWebhookConfiguration{
+		ObjectMeta: meta.ObjectMeta{
+			Name:   config.ValidatingWebhookConfigurationName,
+			Labels: config.KubePolicyAppLabels,
+		},
+		Webhooks: []admregapi.Webhook{
+			constructDebugWebhook(
+				config.ValidatingWebhookName,
+				url,
+				caData),
+		},
+	}
+}
+
 func constructWebhook(name, servicePath string, caData []byte) admregapi.Webhook {
 	return admregapi.Webhook{
 		Name: name,
@ -161,6 +219,34 @@ func constructWebhook(name, servicePath string, caData []byte) admregapi.Webhook
 	}
 }

+func constructDebugWebhook(name, url string, caData []byte) admregapi.Webhook {
+	return admregapi.Webhook{
+		Name: name,
+		ClientConfig: admregapi.WebhookClientConfig{
+			URL:      &url,
+			CABundle: caData,
+		},
+		Rules: []admregapi.RuleWithOperations{
+			admregapi.RuleWithOperations{
+				Operations: []admregapi.OperationType{
+					admregapi.Create,
+				},
+				Rule: admregapi.Rule{
+					APIGroups: []string{
+						"*",
+					},
+					APIVersions: []string{
+						"*",
+					},
+					Resources: []string{
+						"*/*",
+					},
+				},
+			},
+		},
+	}
+}
+
 func (wrc *WebhookRegistrationClient) constructOwner() meta.OwnerReference {
 	kubePolicyDeployment, err := wrc.client.GetKubePolicyDeployment()

--- a/scripts/deploy-controller-debug.sh
+++ b/scripts/deploy-controller-debug.sh
@ -0,0 +1,36 @@
+#!/bin/bash
+
+for i in "$@"
+do
+case $i in
+    --service=*)
+    service="${i#*=}"
+    shift
+    ;;
+    --serverIP=*)
+    serverIP="${i#*=}"
+    shift
+    ;;
+esac
+done
+
+if [ -z "${serverIP}" ]; then
+  echo -e "Please specify '--serverIP' where Kyverno controller runs."
+  exit 1
+fi
+
+if [ -z "${service}" ]; then
+  service="localhost"
+fi
+
+echo "service is $service"
+echo "serverIP is $serverIP"
+
+echo "Generating certificate for the service ${service}..."
+
+certsGenerator="./scripts/generate-self-signed-cert-and-k8secrets-debug.sh"
+chmod +x "${certsGenerator}"
+
+${certsGenerator} "--service=${service}" "--serverIP=${serverIP}" || exit 2
+echo -e "\n### You can build and run kyverno project locally.\n### To check its work, run it with flags --kubeconfig and --serverIP parameters."
+
--- a/scripts/generate-self-signed-cert-and-k8secrets-debug.sh
+++ b/scripts/generate-self-signed-cert-and-k8secrets-debug.sh
@ -0,0 +1,71 @@
+#!/bin/bash
+
+for i in "$@"
+do
+case $i in
+    --service=*)
+    service="${i#*=}"
+    shift
+    ;;
+    --serverIP=*)
+    serverIP="${i#*=}"
+    shift
+    ;;
+esac
+done
+
+destdir="certs"
+if [ ! -d "$destdir" ]; then
+  mkdir ${destdir} || exit 1
+fi
+
+tmpdir=$(mktemp -d)
+cat <<EOF >> ${tmpdir}/csr.conf
+[req]
+req_extensions      = v3_req
+distinguished_name  = req_distinguished_name
+[req_distinguished_name]
+[ v3_req ]
+basicConstraints    = CA:FALSE
+keyUsage            = nonRepudiation, digitalSignature, keyEncipherment
+extendedKeyUsage    = serverAuth
+subjectAltName      = @alt_names
+[alt_names]
+DNS.1               = ${service}
+IP.1                = ${serverIP}
+EOF
+
+if [ ! -z "${service}" ]; then
+    subjectCN="${service}"
+else
+  subjectCN=${serverIP}
+fi
+
+echo "Generating self-signed certificate for CN=${subjectCN}"
+# generate priv key for root CA
+openssl genrsa -out ${destdir}/rootCA.key 4096 
+# generate root CA
+openssl req -x509 -new -nodes -key ${destdir}/rootCA.key -sha256 -days 1024 -out ${destdir}/rootCA.crt -subj "/CN=${subjectCN}"
+# generate priv key
+openssl genrsa -out ${destdir}/webhook.key 4096
+# generate certificate
+openssl req -new -key ${destdir}/webhook.key -out ${destdir}/webhook.csr -subj "/CN=${subjectCN}" -config ${tmpdir}/csr.conf
+# sign the certificate using the root CA
+openssl x509 -req -in ${destdir}/webhook.csr -CA ${destdir}/rootCA.crt -CAkey ${destdir}/rootCA.key -CAcreateserial -out ${destdir}/webhook.crt -days 1024 -sha256 -extensions v3_req  -extfile ${tmpdir}/csr.conf
+
+
+kubectl delete -f definitions/install_debug.yaml 2>/dev/null
+kubectl delete csr,MutatingWebhookConfiguration,ValidatingWebhookConfiguration --all 2>/dev/null
+
+echo "Generating corresponding kubernetes secrets for TLS pair and root CA"
+# create project namespace
+kubectl create ns kyverno
+# create tls pair secret
+kubectl -n kyverno create secret tls kyverno-svc.kyverno.svc.kyverno-tls-pair --cert=${destdir}/webhook.crt --key=${destdir}/webhook.key
+# annotate tls pair secret to specify use of self-signed certificates and check if root CA is created as secret
+kubectl annotate secret kyverno-svc.kyverno.svc.kyverno-tls-pair -n kyverno self-signed-cert=true
+# create root CA secret
+kubectl -n kyverno create secret generic kyverno-svc.kyverno.svc.kyverno-tls-ca --from-file=${destdir}/rootCA.crt
+
+echo "Creating CRD"
+kubectl apply -f definitions/install_debug.yaml