diff --git a/pkg/webhooks/registration.go b/pkg/webhooks/registration.go
index bb2a16c242..be5d79a882 100644
--- a/pkg/webhooks/registration.go
+++ b/pkg/webhooks/registration.go
@@ -11,6 +11,7 @@ import (
 	client "github.com/nirmata/kyverno/pkg/dclient"
 
 	admregapi "k8s.io/api/admissionregistration/v1beta1"
+	errorsapi "k8s.io/apimachinery/pkg/api/errors"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 	admregclient "k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1"
 	rest "k8s.io/client-go/rest"
@@ -86,9 +87,21 @@ func (wrc *WebhookRegistrationClient) Register() error {
 // Register will fail if the config exists, so there is no need to fail on error
 func (wrc *WebhookRegistrationClient) Deregister() {
 	if wrc.serverIP != "" {
-		wrc.registrationClient.MutatingWebhookConfigurations().Delete(config.MutatingWebhookConfigurationDebug, &meta.DeleteOptions{})
-		wrc.registrationClient.ValidatingWebhookConfigurations().Delete(config.ValidatingWebhookConfigurationDebug, &meta.DeleteOptions{})
-		wrc.registrationClient.ValidatingWebhookConfigurations().Delete(config.PolicyValidatingWebhookConfigurationDebug, &meta.DeleteOptions{})
+		if err := wrc.registrationClient.MutatingWebhookConfigurations().Delete(config.MutatingWebhookConfigurationDebug, &meta.DeleteOptions{}); err != nil {
+			if !errorsapi.IsNotFound(err) {
+				glog.Errorf("Failed to deregister debug mutatingWebhookConfiguratinos, err: %v\n", err)
+			}
+		}
+		if err := wrc.registrationClient.ValidatingWebhookConfigurations().Delete(config.ValidatingWebhookConfigurationDebug, &meta.DeleteOptions{}); err != nil {
+			if !errorsapi.IsNotFound(err) {
+				glog.Errorf("Failed to deregister debug validatingWebhookConfiguratinos, err: %v\n", err)
+			}
+		}
+		if err := wrc.registrationClient.ValidatingWebhookConfigurations().Delete(config.PolicyValidatingWebhookConfigurationDebug, &meta.DeleteOptions{}); err != nil {
+			if !errorsapi.IsNotFound(err) {
+				glog.Errorf("Failed to deregister debug policyValidatingWebhookConfiguratinos, err: %v\n", err)
+			}
+		}
 		return
 	}
 
diff --git a/scripts/generate-self-signed-cert-and-k8secrets-debug.sh b/scripts/generate-self-signed-cert-and-k8secrets-debug.sh
index 52f15b756c..49c446cd24 100755
--- a/scripts/generate-self-signed-cert-and-k8secrets-debug.sh
+++ b/scripts/generate-self-signed-cert-and-k8secrets-debug.sh
@@ -55,7 +55,6 @@ openssl x509 -req -in ${destdir}/webhook.csr -CA ${destdir}/rootCA.crt -CAkey ${
 
 
 kubectl delete -f definitions/install_debug.yaml 2>/dev/null
-kubectl delete csr,MutatingWebhookConfiguration,ValidatingWebhookConfiguration --all 2>/dev/null
 
 echo "Generating corresponding kubernetes secrets for TLS pair and root CA"
 # create project namespace