WIP: Structured reconciliation loops for CRDs

Signed-off-by: Gustavo Carvalho <gustavo.carvalho@container-solutions.com>

Makefile

@@ -16,7 +16,8 @@ all: $(addprefix build-,$(ARCH))
 # Image registry for build/push image targets
 export IMAGE_REGISTRY ?= ghcr.io/external-secrets/external-secrets
 
-CRD_DIR     ?= deploy/crds
+BUNDLE_DIR     ?= deploy/crds
+CRD_DIR     ?= config/crds
 
 HELM_DIR    ?= deploy/charts/external-secrets
 TF_DIR ?= terraform
@@ -130,13 +131,14 @@ fmt: lint.check ## Ensure consistent code style
 
 generate: ## Generate code and crds
 	@go run sigs.k8s.io/controller-tools/cmd/controller-gen object:headerFile="hack/boilerplate.go.txt" paths="./..."
-	@go run sigs.k8s.io/controller-tools/cmd/controller-gen crd paths="./..." output:crd:artifacts:config=$(CRD_DIR)
+	@go run sigs.k8s.io/controller-tools/cmd/controller-gen crd paths="./..." output:crd:artifacts:config=$(CRD_DIR)/bases
 # Remove extra header lines in generated CRDs
-	@for i in $(CRD_DIR)/*.yaml; do \
+	@for i in $(CRD_DIR)/bases/*.yaml; do \
   		tail -n +2 <"$$i" >"$$i.bkp" && \
   		cp "$$i.bkp" "$$i" && \
   		rm "$$i.bkp"; \
   	done
+	@kubectl apply -k $(CRD_DIR) --dry-run=client -o yaml > $(BUNDLE_DIR)/bundle.yaml
 	@$(OK) Finished generating deepcopy and crds
 
 # ====================================================================================

config/crds/kustomization.yaml

@@ -0,0 +1,11 @@
+resources:
+- bases/external-secrets.io_clustersecretstores.yaml
+- bases/external-secrets.io_secretstores.yaml
+- bases/external-secrets.io_externalsecrets.yaml
+
+patchesStrategicMerge:
+- patches/webhook_in_externalsecrets.yaml
+- patches/webhook_in_clustersecretstores.yaml
+- patches/webhook_in_secretstores.yaml
+
+configurations: []

config/crds/patches/webhook_in_clustersecretstores.yaml

@@ -0,0 +1,15 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: clustersecretstores.external-secrets.io
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      conversionReviewVersions: ["v1"]
+      clientConfig:
+        caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQ1ekNDQXMrZ0F3SUJBZ0lVWmltdzc0N3NuYkNUVVNXdnhXRVBMR005WVAwd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2dZSXhDekFKQmdOVkJBWVRBa0ZWTVJNd0VRWURWUVFJREFwVGIyMWxMVk4wWVhSbE1TRXdId1lEVlFRSwpEQmhKYm5SbGNtNWxkQ0JYYVdSbmFYUnpJRkIwZVNCTWRHUXhPekE1QmdOVkJBTU1NbVY0ZEdWeWJtRnNMWE5sClkzSmxkSE10ZDJWaWFHOXZheTVrWldaaGRXeDBMbk4yWXk1amJIVnpkR1Z5TG14dlkyRnNNQjRYRFRJeU1ESXcKT1RFeU5UYzBOMW9YRFRJM01ESXdPREV5TlRjME4xb3dnWUl4Q3pBSkJnTlZCQVlUQWtGVk1STXdFUVlEVlFRSQpEQXBUYjIxbExWTjBZWFJsTVNFd0h3WURWUVFLREJoSmJuUmxjbTVsZENCWGFXUm5hWFJ6SUZCMGVTQk1kR1F4Ck96QTVCZ05WQkFNTU1tVjRkR1Z5Ym1Gc0xYTmxZM0psZEhNdGQyVmlhRzl2YXk1a1pXWmhkV3gwTG5OMll5NWoKYkhWemRHVnlMbXh2WTJGc01JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDZQVApVOW1mVEV5aEtQanpCVC9QSnRENU9Zbjg3THNReHQzWFYybWM4UFB2dVZsbWhEdmZwNmljbmZoSlNyNnVTZzEvCnp4dTVsa2VMWno5R0RZM1JIZFNhMDFRUVlIUUFCVk5Ob011VStDMlozeDJVcjV6VnI0RmJLdEd1a1AvWVhJS2EKV1JYcmRxa2hUV1Q4OFExYUR4Zjd0K0R5end6a1hIMmhuRnFrc0JRS3JpL3RBVmsvTWZRaXVOS0k5MG1oenUzWQpMeXdDS0FCRVAvZi9lUzl4MHpJenQrSmNJM0hHVTFYUDROS3I4WjhWaUhFbEd4VWF0M2F2TlZaL1N2SzF3elFSCjlEZXV4WndvOWIwS0F5ellIMmhBdS92ZCtGWkk1Q2Z0WTJNb1lLZWJadm5WT3RPNmdsVjZjb3dtSTlGbGNabEgKdTEvRmdKSUpvZktJM2RhTitRSURBUUFCbzFNd1VUQWRCZ05WSFE0RUZnUVV1Vkk4dlR0aFlGRlN0dzR4Y0VsWQp4SWE2bXRJd0h3WURWUjBqQkJnd0ZvQVV1Vkk4dlR0aFlGRlN0dzR4Y0VsWXhJYTZtdEl3RHdZRFZSMFRBUUgvCkJBVXdBd0VCL3pBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQUlzRkdGU25tUDRZY2JsQU03K3JjZ0pHczhqVEMKbThXSE4wemNvblFXU3R2aGhaSHJLWmtzRGh4aEhkKzNnNk1MYXpuVU1ndWREajZiSGRUendueVlYZEh6MkpzVApvbzg2eGxiODZDZUVucmFFOU9PU2RVWG03a3ZkTVZsTDMzdmFRaEpLbGJHUTdDR1hzSlBjUENIcTlQc21aTlJTCnlqaCtyNmFwZ3Y2S0pLL0tTMW9vck5JSFB0UGVzSVZtS2tuZ0RIdHR0bVFJVjdFSzlQMUFXbVV1WmU2SHE3bUYKY28zbjdpN3lZZ3lTMkQyMkorWElMdVdySFBOU3R2MVZad1VSZVhUeGZ0WGhmT0JUeVNydW4ySmhCc0Y4MXQyRgpOdy8wdEFlS2h1TzFIakZtYTRNOGFYYzdZeGx0emxPZmVpNXV2MEFFRHJkekhaZjk3V2JBSmVNSmdBPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
+        service:
+          namespace: my-external-secrets-namespace
+          name: my-external-secrets-webhook-service
+          path: /convert

config/crds/patches/webhook_in_externalsecrets.yaml

@@ -0,0 +1,15 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: externalsecrets.external-secrets.io
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      conversionReviewVersions: ["v1"]
+      clientConfig:
+        caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQ1ekNDQXMrZ0F3SUJBZ0lVWmltdzc0N3NuYkNUVVNXdnhXRVBMR005WVAwd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2dZSXhDekFKQmdOVkJBWVRBa0ZWTVJNd0VRWURWUVFJREFwVGIyMWxMVk4wWVhSbE1TRXdId1lEVlFRSwpEQmhKYm5SbGNtNWxkQ0JYYVdSbmFYUnpJRkIwZVNCTWRHUXhPekE1QmdOVkJBTU1NbVY0ZEdWeWJtRnNMWE5sClkzSmxkSE10ZDJWaWFHOXZheTVrWldaaGRXeDBMbk4yWXk1amJIVnpkR1Z5TG14dlkyRnNNQjRYRFRJeU1ESXcKT1RFeU5UYzBOMW9YRFRJM01ESXdPREV5TlRjME4xb3dnWUl4Q3pBSkJnTlZCQVlUQWtGVk1STXdFUVlEVlFRSQpEQXBUYjIxbExWTjBZWFJsTVNFd0h3WURWUVFLREJoSmJuUmxjbTVsZENCWGFXUm5hWFJ6SUZCMGVTQk1kR1F4Ck96QTVCZ05WQkFNTU1tVjRkR1Z5Ym1Gc0xYTmxZM0psZEhNdGQyVmlhRzl2YXk1a1pXWmhkV3gwTG5OMll5NWoKYkhWemRHVnlMbXh2WTJGc01JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDZQVApVOW1mVEV5aEtQanpCVC9QSnRENU9Zbjg3THNReHQzWFYybWM4UFB2dVZsbWhEdmZwNmljbmZoSlNyNnVTZzEvCnp4dTVsa2VMWno5R0RZM1JIZFNhMDFRUVlIUUFCVk5Ob011VStDMlozeDJVcjV6VnI0RmJLdEd1a1AvWVhJS2EKV1JYcmRxa2hUV1Q4OFExYUR4Zjd0K0R5end6a1hIMmhuRnFrc0JRS3JpL3RBVmsvTWZRaXVOS0k5MG1oenUzWQpMeXdDS0FCRVAvZi9lUzl4MHpJenQrSmNJM0hHVTFYUDROS3I4WjhWaUhFbEd4VWF0M2F2TlZaL1N2SzF3elFSCjlEZXV4WndvOWIwS0F5ellIMmhBdS92ZCtGWkk1Q2Z0WTJNb1lLZWJadm5WT3RPNmdsVjZjb3dtSTlGbGNabEgKdTEvRmdKSUpvZktJM2RhTitRSURBUUFCbzFNd1VUQWRCZ05WSFE0RUZnUVV1Vkk4dlR0aFlGRlN0dzR4Y0VsWQp4SWE2bXRJd0h3WURWUjBqQkJnd0ZvQVV1Vkk4dlR0aFlGRlN0dzR4Y0VsWXhJYTZtdEl3RHdZRFZSMFRBUUgvCkJBVXdBd0VCL3pBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQUlzRkdGU25tUDRZY2JsQU03K3JjZ0pHczhqVEMKbThXSE4wemNvblFXU3R2aGhaSHJLWmtzRGh4aEhkKzNnNk1MYXpuVU1ndWREajZiSGRUendueVlYZEh6MkpzVApvbzg2eGxiODZDZUVucmFFOU9PU2RVWG03a3ZkTVZsTDMzdmFRaEpLbGJHUTdDR1hzSlBjUENIcTlQc21aTlJTCnlqaCtyNmFwZ3Y2S0pLL0tTMW9vck5JSFB0UGVzSVZtS2tuZ0RIdHR0bVFJVjdFSzlQMUFXbVV1WmU2SHE3bUYKY28zbjdpN3lZZ3lTMkQyMkorWElMdVdySFBOU3R2MVZad1VSZVhUeGZ0WGhmT0JUeVNydW4ySmhCc0Y4MXQyRgpOdy8wdEFlS2h1TzFIakZtYTRNOGFYYzdZeGx0emxPZmVpNXV2MEFFRHJkekhaZjk3V2JBSmVNSmdBPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
+        service:
+          namespace: my-external-secrets-namespace
+          name: my-external-secrets-webhook-service
+          path: /convert

config/crds/patches/webhook_in_secretstores.yaml

@@ -0,0 +1,15 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: secretstores.external-secrets.io
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      conversionReviewVersions: ["v1"]
+      clientConfig:
+        caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQ1ekNDQXMrZ0F3SUJBZ0lVWmltdzc0N3NuYkNUVVNXdnhXRVBMR005WVAwd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2dZSXhDekFKQmdOVkJBWVRBa0ZWTVJNd0VRWURWUVFJREFwVGIyMWxMVk4wWVhSbE1TRXdId1lEVlFRSwpEQmhKYm5SbGNtNWxkQ0JYYVdSbmFYUnpJRkIwZVNCTWRHUXhPekE1QmdOVkJBTU1NbVY0ZEdWeWJtRnNMWE5sClkzSmxkSE10ZDJWaWFHOXZheTVrWldaaGRXeDBMbk4yWXk1amJIVnpkR1Z5TG14dlkyRnNNQjRYRFRJeU1ESXcKT1RFeU5UYzBOMW9YRFRJM01ESXdPREV5TlRjME4xb3dnWUl4Q3pBSkJnTlZCQVlUQWtGVk1STXdFUVlEVlFRSQpEQXBUYjIxbExWTjBZWFJsTVNFd0h3WURWUVFLREJoSmJuUmxjbTVsZENCWGFXUm5hWFJ6SUZCMGVTQk1kR1F4Ck96QTVCZ05WQkFNTU1tVjRkR1Z5Ym1Gc0xYTmxZM0psZEhNdGQyVmlhRzl2YXk1a1pXWmhkV3gwTG5OMll5NWoKYkhWemRHVnlMbXh2WTJGc01JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDZQVApVOW1mVEV5aEtQanpCVC9QSnRENU9Zbjg3THNReHQzWFYybWM4UFB2dVZsbWhEdmZwNmljbmZoSlNyNnVTZzEvCnp4dTVsa2VMWno5R0RZM1JIZFNhMDFRUVlIUUFCVk5Ob011VStDMlozeDJVcjV6VnI0RmJLdEd1a1AvWVhJS2EKV1JYcmRxa2hUV1Q4OFExYUR4Zjd0K0R5end6a1hIMmhuRnFrc0JRS3JpL3RBVmsvTWZRaXVOS0k5MG1oenUzWQpMeXdDS0FCRVAvZi9lUzl4MHpJenQrSmNJM0hHVTFYUDROS3I4WjhWaUhFbEd4VWF0M2F2TlZaL1N2SzF3elFSCjlEZXV4WndvOWIwS0F5ellIMmhBdS92ZCtGWkk1Q2Z0WTJNb1lLZWJadm5WT3RPNmdsVjZjb3dtSTlGbGNabEgKdTEvRmdKSUpvZktJM2RhTitRSURBUUFCbzFNd1VUQWRCZ05WSFE0RUZnUVV1Vkk4dlR0aFlGRlN0dzR4Y0VsWQp4SWE2bXRJd0h3WURWUjBqQkJnd0ZvQVV1Vkk4dlR0aFlGRlN0dzR4Y0VsWXhJYTZtdEl3RHdZRFZSMFRBUUgvCkJBVXdBd0VCL3pBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQUlzRkdGU25tUDRZY2JsQU03K3JjZ0pHczhqVEMKbThXSE4wemNvblFXU3R2aGhaSHJLWmtzRGh4aEhkKzNnNk1MYXpuVU1ndWREajZiSGRUendueVlYZEh6MkpzVApvbzg2eGxiODZDZUVucmFFOU9PU2RVWG03a3ZkTVZsTDMzdmFRaEpLbGJHUTdDR1hzSlBjUENIcTlQc21aTlJTCnlqaCtyNmFwZ3Y2S0pLL0tTMW9vck5JSFB0UGVzSVZtS2tuZ0RIdHR0bVFJVjdFSzlQMUFXbVV1WmU2SHE3bUYKY28zbjdpN3lZZ3lTMkQyMkorWElMdVdySFBOU3R2MVZad1VSZVhUeGZ0WGhmT0JUeVNydW4ySmhCc0Y4MXQyRgpOdy8wdEFlS2h1TzFIakZtYTRNOGFYYzdZeGx0emxPZmVpNXV2MEFFRHJkekhaZjk3V2JBSmVNSmdBPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
+        service:
+          namespace: my-external-secrets-namespace
+          name: my-external-secrets-webhook-service
+          path: /convert

deploy/charts/external-secrets/templates/deployment.yaml

@@ -68,7 +68,7 @@ spec:
           {{- if .Values.webhook.enabled }}
           volumeMounts:
           - name: certs
-            mountPath: /tmp/k8s-webhook-server/serving-certs
+            mountPath: {{ .Values.webhook.certDir }}
           {{- end }}
           ports:
             - containerPort: {{ .Values.prometheus.service.port }}
@@ -89,7 +89,7 @@ spec:
       volumes:
       - name: certs
         secret:
-          secretName: {{ .Values.webhook.secretName }}
+          secretName: {{ include "external-secrets.fullname" . }}-webhook
       {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:

deploy/charts/external-secrets/templates/rbac.yaml

@@ -39,6 +39,24 @@ rules:
     - "get"
     - "list"
     - "watch"
+  - apiGroups:
+    - "apiextensions.k8s.io"
+    resources:
+    - "customresourcedefinitions"
+    verbs:
+    - "get"
+    - "list"
+    - "watch"
+    - "update"
+    - "patch"
+  - apiGroups:
+    - ""
+    resources:
+    - "services"
+    verbs:
+    - "get"
+    - "list"
+    - "watch"
   - apiGroups:
     - ""
     resources:

deploy/charts/external-secrets/templates/webhook-secret.yaml

@@ -0,0 +1,9 @@
+{{- if .Values.webhook.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "external-secrets.fullname" . }}-webhook
+  labels:
+    {{- include "external-secrets.labels" . | nindent 4 }}
+    external-secrets.io/component : webhook
+{{- end }}

deploy/charts/external-secrets/templates/webhook-service.yaml

@@ -2,10 +2,10 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ include "external-secrets.name" . }}-webhook
+  name: {{ include "external-secrets.fullname" . }}-webhook
   labels:
     {{- include "external-secrets.labels" . | nindent 4 }}
-    external-secrets.io/component : conversion-webhook
+    external-secrets.io/component : webhook
 spec:
   type: ClusterIP
   ports:

deploy/charts/external-secrets/values.yaml

@@ -80,8 +80,7 @@ prometheus:
 
 webhook:
   enabled: true
-  secretName: secret-certificate
-
+  certDir: /tmp/k8s-webhook-server/serving-certs
 nodeSelector: {}
 
 tolerations: []

main.go

@@ -29,6 +29,7 @@ import (
 
 	esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1"
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+	"github.com/external-secrets/external-secrets/pkg/controllers/crds"
 	"github.com/external-secrets/external-secrets/pkg/controllers/externalsecret"
 	"github.com/external-secrets/external-secrets/pkg/controllers/secretstore"
 )
@@ -89,6 +90,24 @@ func main() {
 		setupLog.Error(err, "unable to start manager")
 		os.Exit(1)
 	}
+	crds := &crds.Reconciler{
+		Client:                 mgr.GetClient(),
+		Log:                    ctrl.Log.WithName("controllers").WithName("CustomResourceDefinition"),
+		Scheme:                 mgr.GetScheme(),
+		SvcLabels:              map[string]string{"external-secrets.io/component": "webhook"},
+		SecretLabels:           map[string]string{"external-secrets.io/component": "webhook"},
+		CrdResources:           []string{"externalsecrets.external-secrets.io", "clustersecretstores.external-secrets.io", "secretstores.external-secrets.io"},
+		CertDir:                "/tmp/k8s-webhook-server/serving-certs",
+		CAName:                 "external-secrets",
+		CAOrganization:         "external-secrets",
+		RestartOnSecretRefresh: true,
+	}
+	if err := crds.SetupWithManager(mgr, controller.Options{
+		MaxConcurrentReconciles: concurrent,
+	}); err != nil {
+		setupLog.Error(err, errCreateController, "controller", "CustomResourceDefinition")
+		os.Exit(1)
+	}
 
 	if err = (&secretstore.StoreReconciler{
 		Client:          mgr.GetClient(),
@@ -122,35 +141,36 @@ func main() {
 		setupLog.Error(err, errCreateController, "controller", "ExternalSecret")
 		os.Exit(1)
 	}
-
-	if err = (&esv1beta1.ExternalSecret{}).SetupWebhookWithManager(mgr); err != nil {
-		setupLog.Error(err, errCreateWebhook, "webhook", "ExternalSecret-v1beta1")
-		os.Exit(1)
+	if crtsReady := crds.EnsureCertsMounted(); crtsReady {
+		if err = (&esv1beta1.ExternalSecret{}).SetupWebhookWithManager(mgr); err != nil {
+			setupLog.Error(err, errCreateWebhook, "webhook", "ExternalSecret-v1beta1")
+			os.Exit(1)
+		}
+		if err = (&esv1beta1.SecretStore{}).SetupWebhookWithManager(mgr); err != nil {
+			setupLog.Error(err, errCreateWebhook, "webhook", "SecretStore-v1beta1")
+			os.Exit(1)
+		}
+		if err = (&esv1beta1.ClusterSecretStore{}).SetupWebhookWithManager(mgr); err != nil {
+			setupLog.Error(err, errCreateWebhook, "webhook", "ClusterSecretStore-v1beta1")
+			os.Exit(1)
+		}
+		if err = (&esv1alpha1.ExternalSecret{}).SetupWebhookWithManager(mgr); err != nil {
+			setupLog.Error(err, errCreateWebhook, "webhook", "ExternalSecret-v1alpha1")
+			os.Exit(1)
+		}
+		if err = (&esv1alpha1.SecretStore{}).SetupWebhookWithManager(mgr); err != nil {
+			setupLog.Error(err, errCreateWebhook, "webhook", "SecretStore-v1alpha1")
+			os.Exit(1)
+		}
+		if err = (&esv1alpha1.ClusterSecretStore{}).SetupWebhookWithManager(mgr); err != nil {
+			setupLog.Error(err, errCreateWebhook, "webhook", "ClusterSecretStore-v1alpha1")
+			os.Exit(1)
+		}
 	}
-	if err = (&esv1beta1.SecretStore{}).SetupWebhookWithManager(mgr); err != nil {
-		setupLog.Error(err, errCreateWebhook, "webhook", "SecretStore-v1beta1")
-		os.Exit(1)
-	}
-	if err = (&esv1beta1.ClusterSecretStore{}).SetupWebhookWithManager(mgr); err != nil {
-		setupLog.Error(err, errCreateWebhook, "webhook", "ClusterSecretStore-v1beta1")
-		os.Exit(1)
-	}
-	if err = (&esv1alpha1.ExternalSecret{}).SetupWebhookWithManager(mgr); err != nil {
-		setupLog.Error(err, errCreateWebhook, "webhook", "ExternalSecret-v1alpha1")
-		os.Exit(1)
-	}
-	if err = (&esv1alpha1.SecretStore{}).SetupWebhookWithManager(mgr); err != nil {
-		setupLog.Error(err, errCreateWebhook, "webhook", "SecretStore-v1alpha1")
-		os.Exit(1)
-	}
-	if err = (&esv1alpha1.ClusterSecretStore{}).SetupWebhookWithManager(mgr); err != nil {
-		setupLog.Error(err, errCreateWebhook, "webhook", "ClusterSecretStore-v1alpha1")
-		os.Exit(1)
-	}
-
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)
 	}
+
 }

pkg/controllers/crds/crds_controller.go

@@ -0,0 +1,468 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crds
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/base64"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"math/big"
+	"os"
+	"time"
+
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+)
+
+const (
+	certName               = "tls.crt"
+	keyName                = "tls.key"
+	caCertName             = "ca.crt"
+	caKeyName              = "ca.key"
+	rotationCheckFrequency = 12 * time.Hour
+	certValidityDuration   = 10 * 365 * 24 * time.Hour
+	lookaheadInterval      = 90 * 24 * time.Hour
+)
+
+type WebhookType int
+
+const (
+	//ValidatingWebhook indicates the webhook is a ValidatingWebhook
+	Validating WebhookType = iota
+	//MutingWebhook indicates the webhook is a MutatingWebhook
+	Mutating
+	//CRDConversionWebhook indicates the webhook is a conversion webhook
+	CRDConversion
+	//APIServiceWebhook indicates the webhook is an extension API server
+	APIService
+)
+
+type Reconciler struct {
+	client.Client
+	Log                    logr.Logger
+	Scheme                 *runtime.Scheme
+	recorder               record.EventRecorder
+	SvcLabels              map[string]string
+	SecretLabels           map[string]string
+	CrdResources           []string
+	CertDir                string
+	dnsName                string
+	CAName                 string
+	CAOrganization         string
+	RestartOnSecretRefresh bool
+}
+
+type WebhookInfo struct {
+	//Name is the name of the webhook for a validating or mutating webhook, or the CRD name in case of a CRD conversion webhook
+	Name string
+	Type WebhookType
+}
+
+func contains(s []string, e string) bool {
+	for _, a := range s {
+		if a == e {
+			return true
+		}
+	}
+	return false
+}
+func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := r.Log.WithValues("CustomResourceDefinition", req.NamespacedName)
+	if contains(r.CrdResources, req.NamespacedName.Name) {
+		err := r.updateCRD(ctx, req)
+		if err != nil {
+			log.Error(err, "failed to inject conversion webhook")
+			return ctrl.Result{}, err
+		}
+	}
+	return ctrl.Result{}, nil
+}
+
+func (r *Reconciler) ConvertToWebhookInfo() []WebhookInfo {
+	info := make([]WebhookInfo, len(r.CrdResources))
+	for p, v := range r.CrdResources {
+		r := WebhookInfo{
+			Name: v,
+			Type: CRDConversion,
+		}
+		info[p] = r
+	}
+	return info
+}
+
+func (r *Reconciler) SetupWithManager(mgr ctrl.Manager, opts controller.Options) error {
+	crdGVK := schema.GroupVersionKind{Group: "apiextensions.k8s.io", Version: "v1", Kind: "CustomResourceDefinition"}
+	res := &unstructured.Unstructured{}
+	res.SetGroupVersionKind(crdGVK)
+	r.recorder = mgr.GetEventRecorderFor("custom-resource-definition")
+	return ctrl.NewControllerManagedBy(mgr).
+		WithOptions(opts).
+		For(res).
+		Complete(r)
+}
+
+func (r *Reconciler) updateCRD(ctx context.Context, req ctrl.Request) error {
+	crdGVK := schema.GroupVersionKind{Group: "apiextensions.k8s.io", Version: "v1", Kind: "CustomResourceDefinition"}
+
+	svcList := corev1.ServiceList{}
+	err := r.List(context.Background(), &svcList, client.MatchingLabels(r.SvcLabels))
+	if err != nil {
+		return err
+	}
+	if len(svcList.Items) != 1 {
+		return errors.New("multiple services match labels")
+	}
+	secretList := corev1.SecretList{}
+	err = r.List(context.Background(), &secretList, client.MatchingLabels(r.SecretLabels))
+	if err != nil {
+		return err
+	}
+	if len(secretList.Items) != 1 {
+		return errors.New("multiple secrets match labels")
+	}
+	updatedResource := &unstructured.Unstructured{}
+	updatedResource.SetGroupVersionKind(crdGVK)
+	if err := r.Get(ctx, req.NamespacedName, updatedResource); err != nil {
+		return err
+	}
+	if err := injectSvcToConversionWebhook(updatedResource, &svcList.Items[0]); err != nil {
+		return err
+	}
+	r.dnsName = fmt.Sprintf("%v.%v.svc", svcList.Items[0].Name, svcList.Items[0].Namespace)
+	need, err := r.refreshCertIfNeeded(&secretList.Items[0])
+	if err != nil {
+		return err
+	}
+	if need {
+		artifacts, err := buildArtifactsFromSecret(&secretList.Items[0])
+		if err != nil {
+			return err
+		}
+		if err := injectCertToConversionWebhook(updatedResource, artifacts.CertPEM); err != nil {
+			return err
+		}
+	}
+	if err := r.Update(ctx, updatedResource); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (r *Reconciler) EnsureCertsMounted() bool {
+	certFile := r.CertDir + "/" + certName
+	_, err := os.Stat(certFile)
+	return err == nil
+}
+
+func injectSvcToConversionWebhook(crd *unstructured.Unstructured, service *corev1.Service) error {
+	_, found, err := unstructured.NestedMap(crd.Object, "spec", "conversion", "webhook", "clientConfig")
+	if err != nil {
+		return err
+	}
+	if !found {
+		return errors.New("`conversion.webhook.clientConfig` field not found in CustomResourceDefinition")
+	}
+	if err := unstructured.SetNestedField(crd.Object, service.Name, "spec", "conversion", "webhook", "clientConfig", "service", "name"); err != nil {
+		return err
+	}
+	if err := unstructured.SetNestedField(crd.Object, service.Namespace, "spec", "conversion", "webhook", "clientConfig", "service", "namespace"); err != nil {
+		return err
+	}
+	return nil
+}
+
+func injectCertToConversionWebhook(crd *unstructured.Unstructured, certPem []byte) error {
+	_, found, err := unstructured.NestedMap(crd.Object, "spec", "conversion", "webhook", "clientConfig")
+	if err != nil {
+		return err
+	}
+	if !found {
+		return errors.New("`conversion.webhook.clientConfig` field not found in CustomResourceDefinition")
+	}
+	if err := unstructured.SetNestedField(crd.Object, base64.StdEncoding.EncodeToString(certPem), "spec", "conversion", "webhook", "clientConfig", "caBundle"); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type KeyPairArtifacts struct {
+	Cert    *x509.Certificate
+	Key     *rsa.PrivateKey
+	CertPEM []byte
+	KeyPEM  []byte
+}
+
+func populateSecret(cert, key []byte, caArtifacts *KeyPairArtifacts, secret *corev1.Secret) {
+	if secret.Data == nil {
+		secret.Data = make(map[string][]byte)
+	}
+	secret.Data[caCertName] = caArtifacts.CertPEM
+	secret.Data[caKeyName] = caArtifacts.KeyPEM
+	secret.Data[certName] = cert
+	secret.Data[keyName] = key
+}
+
+func ValidCert(caCert, cert, key []byte, dnsName string, at time.Time) (bool, error) {
+	if len(caCert) == 0 || len(cert) == 0 || len(key) == 0 {
+		return false, errors.New("empty cert")
+	}
+
+	pool := x509.NewCertPool()
+	caDer, _ := pem.Decode(caCert)
+	if caDer == nil {
+		return false, errors.New("bad CA cert")
+	}
+	cac, err := x509.ParseCertificate(caDer.Bytes)
+	if err != nil {
+		return false, err
+	}
+	pool.AddCert(cac)
+
+	_, err = tls.X509KeyPair(cert, key)
+	if err != nil {
+		return false, err
+	}
+
+	b, _ := pem.Decode(cert)
+	if b == nil {
+		return false, err
+	}
+
+	crt, err := x509.ParseCertificate(b.Bytes)
+	if err != nil {
+		return false, err
+	}
+	_, err = crt.Verify(x509.VerifyOptions{
+		DNSName:     dnsName,
+		Roots:       pool,
+		CurrentTime: at,
+	})
+	if err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+func lookaheadTime() time.Time {
+	return time.Now().Add(lookaheadInterval)
+}
+
+func (r *Reconciler) validServerCert(caCert, cert, key []byte) bool {
+	valid, err := ValidCert(caCert, cert, key, r.dnsName, lookaheadTime())
+	if err != nil {
+		return false
+	}
+	return valid
+}
+
+func (r *Reconciler) validCACert(cert, key []byte) bool {
+	valid, err := ValidCert(cert, cert, key, r.CAName, lookaheadTime())
+	if err != nil {
+		return false
+	}
+	return valid
+}
+
+func (r *Reconciler) refreshCertIfNeeded(secret *corev1.Secret) (bool, error) {
+	if secret.Data == nil || !r.validCACert(secret.Data[caCertName], secret.Data[caKeyName]) {
+		//crLog.Info("refreshing CA and server certs")
+		if err := r.refreshCerts(true, secret); err != nil {
+			//crLog.Error(err, "could not refresh CA and server certs")
+			return false, nil
+		}
+		//crLog.Info("server certs refreshed")
+		if r.RestartOnSecretRefresh {
+			//crLog.Info("Secrets have been updated; exiting so pod can be restarted (This behaviour can be changed with the option RestartOnSecretRefresh)")
+			os.Exit(0)
+		}
+		return true, nil
+	}
+	// make sure our reconciler is initialized on startup (either this or the above refreshCerts() will call this)
+	if !r.validServerCert(secret.Data[caCertName], secret.Data[certName], secret.Data[keyName]) {
+		//crLog.Info("refreshing server certs")
+		if err := r.refreshCerts(false, secret); err != nil {
+			//crLog.Error(err, "could not refresh server certs")
+			return false, nil
+		}
+		//crLog.Info("server certs refreshed")
+		if r.RestartOnSecretRefresh {
+			//crLog.Info("Secrets have been updated; exiting so pod can be restarted (This behaviour can be changed with the option RestartOnSecretRefresh)")
+			os.Exit(0)
+		}
+		return true, nil
+	}
+	//crLog.Info("no cert refresh needed")
+	return true, nil
+}
+
+func (r *Reconciler) refreshCerts(refreshCA bool, secret *corev1.Secret) error {
+	var caArtifacts *KeyPairArtifacts
+	now := time.Now()
+	begin := now.Add(-1 * time.Hour)
+	end := now.Add(certValidityDuration)
+	if refreshCA {
+		var err error
+		caArtifacts, err = r.CreateCACert(begin, end)
+		if err != nil {
+			return err
+		}
+	} else {
+		var err error
+		caArtifacts, err = buildArtifactsFromSecret(secret)
+		if err != nil {
+			return err
+		}
+	}
+	cert, key, err := r.CreateCertPEM(caArtifacts, begin, end)
+	if err != nil {
+		return err
+	}
+	if err := r.writeSecret(cert, key, caArtifacts, secret); err != nil {
+		return err
+	}
+	return nil
+}
+
+func buildArtifactsFromSecret(secret *corev1.Secret) (*KeyPairArtifacts, error) {
+	caPem, ok := secret.Data[caCertName]
+	if !ok {
+		return nil, fmt.Errorf("cert secret is not well-formed, missing %s", caCertName)
+	}
+	keyPem, ok := secret.Data[caKeyName]
+	if !ok {
+		return nil, fmt.Errorf("cert secret is not well-formed, missing %s", caKeyName)
+	}
+	caDer, _ := pem.Decode(caPem)
+	if caDer == nil {
+		return nil, errors.New("bad CA cert")
+	}
+	caCert, err := x509.ParseCertificate(caDer.Bytes)
+	if err != nil {
+		return nil, err
+	}
+	keyDer, _ := pem.Decode(keyPem)
+	if keyDer == nil {
+		return nil, err
+	}
+	key, err := x509.ParsePKCS1PrivateKey(keyDer.Bytes)
+	if err != nil {
+		return nil, err
+	}
+	return &KeyPairArtifacts{
+		Cert:    caCert,
+		CertPEM: caPem,
+		KeyPEM:  keyPem,
+		Key:     key,
+	}, nil
+}
+
+func (r *Reconciler) CreateCACert(begin, end time.Time) (*KeyPairArtifacts, error) {
+	templ := &x509.Certificate{
+		SerialNumber: big.NewInt(0),
+		Subject: pkix.Name{
+			CommonName:   r.CAName,
+			Organization: []string{r.CAOrganization},
+		},
+		DNSNames: []string{
+			r.CAName,
+		},
+		NotBefore:             begin,
+		NotAfter:              end,
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+	}
+	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return nil, err
+	}
+	der, err := x509.CreateCertificate(rand.Reader, templ, templ, key.Public(), key)
+	if err != nil {
+		return nil, err
+	}
+	certPEM, keyPEM, err := pemEncode(der, key)
+	if err != nil {
+		return nil, err
+	}
+	cert, err := x509.ParseCertificate(der)
+	if err != nil {
+		return nil, err
+	}
+
+	return &KeyPairArtifacts{Cert: cert, Key: key, CertPEM: certPEM, KeyPEM: keyPEM}, nil
+}
+
+func (r *Reconciler) CreateCertPEM(ca *KeyPairArtifacts, begin, end time.Time) ([]byte, []byte, error) {
+	templ := &x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject: pkix.Name{
+			CommonName: r.dnsName,
+		},
+		DNSNames: []string{
+			r.dnsName,
+		},
+		NotBefore:             begin,
+		NotAfter:              end,
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+	}
+	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return nil, nil, err
+	}
+	der, err := x509.CreateCertificate(rand.Reader, templ, ca.Cert, key.Public(), ca.Key)
+	if err != nil {
+		return nil, nil, err
+	}
+	certPEM, keyPEM, err := pemEncode(der, key)
+	if err != nil {
+		return nil, nil, err
+	}
+	return certPEM, keyPEM, nil
+}
+
+func pemEncode(certificateDER []byte, key *rsa.PrivateKey) ([]byte, []byte, error) {
+	certBuf := &bytes.Buffer{}
+	if err := pem.Encode(certBuf, &pem.Block{Type: "CERTIFICATE", Bytes: certificateDER}); err != nil {
+		return nil, nil, err
+	}
+	keyBuf := &bytes.Buffer{}
+	if err := pem.Encode(keyBuf, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}); err != nil {
+		return nil, nil, err
+	}
+	return certBuf.Bytes(), keyBuf.Bytes(), nil
+}
+
+func (r *Reconciler) writeSecret(cert, key []byte, caArtifacts *KeyPairArtifacts, secret *corev1.Secret) error {
+	populateSecret(cert, key, caArtifacts, secret)
+	return r.Update(context.Background(), secret)
+}