From 254be4c1d34ca9a6aef451fadc6108aa2c3c483c Mon Sep 17 00:00:00 2001 From: Kumar Mallikarjuna Date: Thu, 7 Oct 2021 04:42:07 +0530 Subject: [PATCH] Leader Election for initContainer (#2489) * Local build Signed-off-by: Kumar Mallikarjuna * Leader Election for initContainer Signed-off-by: Kumar Mallikarjuna * Lease deletion Signed-off-by: Kumar Mallikarjuna * Use wrc client Signed-off-by: Kumar Mallikarjuna * log error out Signed-off-by: ShutingZhao Co-authored-by: ShutingZhao --- Makefile | 5 +++ cmd/initContainer/localDockerfile | 4 ++ cmd/initContainer/main.go | 73 +++++++++++++++++++++++++------ pkg/webhookconfig/registration.go | 5 +++ 4 files changed, 73 insertions(+), 14 deletions(-) create mode 100644 cmd/initContainer/localDockerfile diff --git a/Makefile b/Makefile index d3f2e1a122..5d31c2aaa2 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,11 @@ docker-push-initContainer: @docker buildx build --file $(PWD)/$(INITC_PATH)/Dockerfile --progress plane --push --platform linux/arm64,linux/amd64 --tag $(REPO)/$(INITC_IMAGE):$(IMAGE_TAG) . --build-arg LD_FLAGS=$(LD_FLAGS) @docker buildx build --file $(PWD)/$(INITC_PATH)/Dockerfile --progress plane --push --platform linux/arm64,linux/amd64 --tag $(REPO)/$(INITC_IMAGE):latest . --build-arg LD_FLAGS=$(LD_FLAGS) +docker-build-initContainer-local: + CGO_ENABLED=0 GOOS=linux go build -o $(PWD)/$(INITC_PATH)/kyvernopre -tags $(TAGS) -ldflags=$(LD_FLAGS) $(PWD)/$(INITC_PATH)/main.go + @docker build -f $(PWD)/$(INITC_PATH)/localDockerfile -t $(REPO)/$(INITC_IMAGE):$(IMAGE_TAG) $(PWD)/$(INITC_PATH) + @docker tag $(REPO)/$(INITC_IMAGE):$(IMAGE_TAG) $(REPO)/$(INITC_IMAGE):latest + ################################## # KYVERNO CONTAINER ################################## diff --git a/cmd/initContainer/localDockerfile b/cmd/initContainer/localDockerfile new file mode 100644 index 0000000000..166b114831 --- /dev/null +++ b/cmd/initContainer/localDockerfile @@ -0,0 +1,4 @@ +FROM scratch +ADD kyvernopre /kyvernopre +USER 10001 +ENTRYPOINT ["/kyvernopre"] diff --git a/cmd/initContainer/main.go b/cmd/initContainer/main.go index cbbc6e7771..d88b3cdcbb 100644 --- a/cmd/initContainer/main.go +++ b/cmd/initContainer/main.go @@ -14,8 +14,10 @@ import ( kyvernoclient "github.com/kyverno/kyverno/pkg/client/clientset/versioned" "github.com/kyverno/kyverno/pkg/config" client "github.com/kyverno/kyverno/pkg/dclient" + "github.com/kyverno/kyverno/pkg/leaderelection" "github.com/kyverno/kyverno/pkg/signal" "github.com/kyverno/kyverno/pkg/utils" + coord "k8s.io/api/coordination/v1" "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" rest "k8s.io/client-go/rest" @@ -95,29 +97,72 @@ func main() { {clusterPolicyViolation, ""}, } + kubeClientLeaderElection, err := utils.NewKubeClient(clientConfig) + if err != nil { + setupLog.Error(err, "Failed to create kubernetes client") + os.Exit(1) + } + + ctx, cancel := context.WithCancel(context.Background()) + + go func() { + <-stopCh + cancel() + }() + done := make(chan struct{}) defer close(done) failure := false - // use pipline to pass request to cleanup resources - // generate requests - in := gen(done, stopCh, requests...) - // process requests - // processing routine count : 2 - p1 := process(client, pclient, done, stopCh, in) - p2 := process(client, pclient, done, stopCh, in) - // merge results from processing routines - for err := range merge(done, stopCh, p1, p2) { + + run := func() { + _, err := kubeClientLeaderElection.CoordinationV1().Leases(getKyvernoNameSpace()).Get(ctx, "kyvernopre-lock", v1.GetOptions{}) + if err != nil { - failure = true - log.Log.Error(err, "failed to cleanup resource") + log.Log.Info("Lease 'kyvernopre-lock' not found. Starting clean-up...") + } else { + log.Log.Info("Clean-up complete. Leader exiting...") + os.Exit(0) } + + // use pipline to pass request to cleanup resources + // generate requests + in := gen(done, stopCh, requests...) + // process requests + // processing routine count : 2 + p1 := process(client, pclient, done, stopCh, in) + p2 := process(client, pclient, done, stopCh, in) + // merge results from processing routines + for err := range merge(done, stopCh, p1, p2) { + if err != nil { + failure = true + log.Log.Error(err, "failed to cleanup resource") + } + } + // if there is any failure then we fail process + if failure { + log.Log.Info("failed to cleanup prior configurations") + os.Exit(1) + } + + lease := coord.Lease{} + lease.ObjectMeta.Name = "kyvernopre-lock" + _, err = kubeClientLeaderElection.CoordinationV1().Leases(getKyvernoNameSpace()).Create(ctx, &lease, v1.CreateOptions{}) + if err != nil { + log.Log.Info("Failed to create lease 'kyvernopre-lock'") + } + + log.Log.Info("Clean-up complete. Leader exiting...") + + os.Exit(0) } - // if there is any failure then we fail process - if failure { - log.Log.Info("failed to cleanup prior configurations") + le, err := leaderelection.New("kyvernopre", getKyvernoNameSpace(), kubeClientLeaderElection, run, nil, log.Log.WithName("kyvernopre/LeaderElection")) + if err != nil { + setupLog.Error(err, "failed to elect a leader") os.Exit(1) } + + le.Run(ctx) } func executeRequest(client *client.Client, pclient *kyvernoclient.Clientset, req request) error { diff --git a/pkg/webhookconfig/registration.go b/pkg/webhookconfig/registration.go index 9b6978dfa6..e1c1f0f918 100644 --- a/pkg/webhookconfig/registration.go +++ b/pkg/webhookconfig/registration.go @@ -172,6 +172,11 @@ func (wrc *Register) Remove(cleanUp chan<- struct{}) { wrc.removeWebhookConfigurations() wrc.removeSecrets() + err := wrc.client.DeleteResource("coordination.k8s.io/v1", "Lease", config.KyvernoNamespace, "kyvernopre-lock", false) + if err != nil && errorsapi.IsNotFound(err) { + wrc.log.WithName("cleanup").Error(err, "failed to clean up Lease lock") + } + } // +deprecated