mirrors/kyverno
1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2024-12-14 11:57:48 +00:00
Code Activity

Fix Kyverno crash when CRD is not installed (#1353)

Browse source 
* ignore Kyverno CRDs existence check when server is not available

* clean up cluster / reportChangeRequest

* resolve PR comments
This commit is contained in:
shuting 2020-12-03 19:19:36 -08:00 committed by GitHub
parent 9cae63ea03
commit 630a9cc94c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 151 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
charts/kyverno/templates/deployment.yaml
View file

@ -50,6 +50,11 @@ spec:
capabilities:
drop:
- all
env:
- name: KYVERNO_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
containers:
- name: kyverno
image: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}

97
cmd/initContainer/main.go
View file

@ -29,12 +29,14 @@ var (
)
const (
mutatingWebhookConfigKind string = "MutatingWebhookConfiguration"
validatingWebhookConfigKind string = "ValidatingWebhookConfiguration"
policyReportKind string = "PolicyReport"
clusterPolicyReportKind string = "ClusterPolicyReport"
policyViolation string = "PolicyViolation"
clusterPolicyViolation string = "ClusterPolicyViolation"
mutatingWebhookConfigKind string = "MutatingWebhookConfiguration"
validatingWebhookConfigKind string = "ValidatingWebhookConfiguration"
policyReportKind string = "PolicyReport"
clusterPolicyReportKind string = "ClusterPolicyReport"
reportChangeRequestKind string = "ReportChangeRequest"
clusterReportChangeRequestKind string = "ClusterReportChangeRequest"
policyViolation string = "PolicyViolation"
clusterPolicyViolation string = "ClusterPolicyViolation"
)
func main() {
@ -70,20 +72,23 @@ func main() {
}
requests := []request{
// Resource
{validatingWebhookConfigKind, config.ValidatingWebhookConfigurationName},
{validatingWebhookConfigKind, config.ValidatingWebhookConfigurationDebugName},
{mutatingWebhookConfigKind, config.MutatingWebhookConfigurationName},
{mutatingWebhookConfigKind, config.MutatingWebhookConfigurationDebugName},
// Policy
{validatingWebhookConfigKind, config.PolicyValidatingWebhookConfigurationName},
{validatingWebhookConfigKind, config.PolicyValidatingWebhookConfigurationDebugName},
{mutatingWebhookConfigKind, config.PolicyMutatingWebhookConfigurationName},
{mutatingWebhookConfigKind, config.PolicyMutatingWebhookConfigurationDebugName},
// policy report
{policyReportKind, ""},
{clusterPolicyReportKind, ""},
// clean up policy violation
{reportChangeRequestKind, ""},
{clusterReportChangeRequestKind, ""},
// clean up policy violation CRD
{policyViolation, ""},
{clusterPolicyViolation, ""},
}
@ -120,6 +125,10 @@ func executeRequest(client *client.Client, req request) error {
return removePolicyReport(client, req.kind)
case clusterPolicyReportKind:
return removeClusterPolicyReport(client, req.kind)
case reportChangeRequestKind:
return removeReportChangeRequest(client, req.kind)
case clusterReportChangeRequestKind:
return removeClusterReportChangeRequest(client, req.kind)
case policyViolation, clusterPolicyViolation:
return removeViolationCRD(client)
}
@ -253,7 +262,7 @@ func removeClusterPolicyReport(client *client.Client, kind string) error {
cpolrs, err := client.ListResource("", kind, "", nil)
if err != nil && !errors.IsNotFound(err) {
logger.Error(err, "failed to list clusterPolicyReport")
return err
return nil
}
for _, cpolr := range cpolrs.Items {
@ -276,20 +285,63 @@ func removePolicyReport(client *client.Client, kind string) error {
}
// name of namespace policy report follows the name convention
// policyreport-ns-<namespace name>
// pr-ns-<namespace name>
for _, ns := range namespaces.Items {
reportName := fmt.Sprintf("pr-ns-%s", ns.GetName())
err := client.DeleteResource("", kind, ns.GetName(), reportName, false)
if err != nil && !errors.IsNotFound(err) {
logger.Error(err, "failed to delete policyReport", "name", reportName)
} else {
logger.Info("successfully cleaned up PolicyReport", "name", reportName)
reportNames := []string{
fmt.Sprintf("policyreport-ns-%s", ns.GetName()),
fmt.Sprintf("pr-ns-%s", ns.GetName()),
}
for _, reportName := range reportNames {
err := client.DeleteResource("", kind, ns.GetName(), reportName, false)
if err != nil && !errors.IsNotFound(err) {
logger.Error(err, "failed to delete resource", "kind", kind, "name", reportName)
} else {
logger.Info("successfully cleaned up resource", "kind", kind, "name", reportName)
}
}
}
return nil
}
func removeReportChangeRequest(client *client.Client, kind string) error {
logger := log.Log.WithName("removeReportChangeRequest")
ns := getKyvernoNameSpace()
rcrList, err := client.ListResource("", kind, ns, nil)
if err != nil && !errors.IsNotFound(err) {
logger.Error(err, "failed to list reportChangeRequest")
return nil
}
for _, rcr := range rcrList.Items {
if err := client.DeleteResource(rcr.GetAPIVersion(), rcr.GetKind(), rcr.GetNamespace(), rcr.GetName(), false); err != nil {
logger.Error(err, "failed to delete reportChangeRequest", "name", rcr.GetName())
} else {
logger.Info("successfully cleaned up reportChangeRequest", "name", rcr.GetName())
}
}
return nil
}
func removeClusterReportChangeRequest(client *client.Client, kind string) error {
crcrList, err := client.ListResource("", kind, "", nil)
if err != nil && !errors.IsNotFound(err) {
logger.Error(err, "failed to list clusterReportChangeRequest")
return nil
}
for _, crcr := range crcrList.Items {
if err := client.DeleteResource(crcr.GetAPIVersion(), crcr.GetKind(), "", crcr.GetName(), false); err != nil {
logger.Error(err, "failed to delete clusterReportChangeRequest", "name", crcr.GetName())
} else {
logger.Info("successfully cleaned up clusterReportChangeRequest")
}
}
return nil
}
func removeViolationCRD(client *client.Client) error {
if err := client.DeleteResource("", "CustomResourceDefinition", "", "policyviolations.kyverno.io", false); err != nil {
if !errors.IsNotFound(err) {
@ -304,3 +356,12 @@ func removeViolationCRD(client *client.Client) error {
}
return nil
}
// getKubePolicyNameSpace - setting default KubePolicyNameSpace
func getKyvernoNameSpace() string {
kyvernoNamespace := os.Getenv("KYVERNO_NAMESPACE")
if kyvernoNamespace == "" {
kyvernoNamespace = "kyverno"
}
return kyvernoNamespace
}

12
cmd/kyverno/main.go
View file

@ -228,7 +228,7 @@ func main() {
// GENERATE CONTROLLER
// - applies generate rules on resources based on generate requests created by webhook
grc := generate.NewController(
grc, err := generate.NewController(
pclient,
client,
pInformer.Kyverno().V1().ClusterPolicies(),
@ -240,10 +240,14 @@ func main() {
configData,
rCache,
)
if err != nil {
setupLog.Error(err, "Failed to create generate controller")
os.Exit(1)
}
// GENERATE REQUEST CLEANUP
// -- cleans up the generate requests that have not been processed(i.e. state = [Pending, Failed]) for more than defined timeout
grcc := generatecleanup.NewController(
grcc, err := generatecleanup.NewController(
pclient,
client,
pInformer.Kyverno().V1().ClusterPolicies(),
@ -251,6 +255,10 @@ func main() {
kubedynamicInformer,
log.Log.WithName("GenerateCleanUpController"),
)
if err != nil {
setupLog.Error(err, "Failed to create generate cleanup controller")
os.Exit(1)
}
pCacheController := policycache.NewPolicyCacheController(
pInformer.Kyverno().V1().ClusterPolicies(),

8
pkg/auth/auth.go
View file

@ -43,7 +43,11 @@ func NewCanI(client *client.Client, kind, namespace, verb string, log logr.Logge
func (o *CanIOptions) RunAccessCheck() (bool, error) {
// get GroupVersionResource from RESTMapper
// get GVR from kind
gvr := o.client.DiscoveryClient.GetGVRFromKind(o.kind)
gvr, err := o.client.DiscoveryClient.GetGVRFromKind(o.kind)
if err != nil {
return false, fmt.Errorf("failed to get GVR for kind %s", o.kind)
}
if reflect.DeepEqual(gvr, schema.GroupVersionResource{}) {
// cannot find GVR
return false, fmt.Errorf("failed to get the Group Version Resource for kind %s", o.kind)
@ -92,7 +96,7 @@ func (o *CanIOptions) RunAccessCheck() (bool, error) {
logger.Info("field not found", "field", "status.reason")
}
// status.evaluationError
evaluationError, ok, err := unstructured.NestedString(resp.Object, "status", "evaludationError")
evaluationError, ok, err := unstructured.NestedString(resp.Object, "status", "evaluationError")
if !ok {
if err != nil {
logger.Error(err, "failed to get the field", "field", "status.evaluationError")

2
pkg/dclient/certificates.go
View file

@ -44,7 +44,7 @@ func (c *Client) buildTLSPemPair(props tls.CertificateProps, fqdncn bool) (*tls.
}
if err := c.WriteCACertToSecret(caPEM, props); err != nil {
return nil, err
return nil, fmt.Errorf("failed to write CA cert to secret: %v", err)
}
return tls.GenerateCertPem(caCert, props, fqdncn)
}

13
pkg/dclient/client.go
View file

@ -118,7 +118,8 @@ func (c *Client) getResourceInterface(apiVersion string, kind string, namespace
// Keep this a stateful as the resource list will be based on the kubernetes version we connect to
func (c *Client) getGroupVersionMapper(apiVersion string, kind string) schema.GroupVersionResource {
if apiVersion == "" {
return c.DiscoveryClient.GetGVRFromKind(kind)
gvr, _ := c.DiscoveryClient.GetGVRFromKind(kind)
return gvr
}
return c.DiscoveryClient.GetGVRFromAPIVersionKind(apiVersion, kind)
@ -227,7 +228,7 @@ func convertToCSR(obj *unstructured.Unstructured) (*certificates.CertificateSign
//IDiscovery provides interface to mange Kind and GVR mapping
type IDiscovery interface {
FindResource(apiVersion string, kind string) (*meta.APIResource, schema.GroupVersionResource, error)
GetGVRFromKind(kind string) schema.GroupVersionResource
GetGVRFromKind(kind string) (schema.GroupVersionResource, error)
GetGVRFromAPIVersionKind(apiVersion string, kind string) schema.GroupVersionResource
GetServerVersion() (*version.Info, error)
OpenAPISchema() (*openapi_v2.Document, error)
@ -275,18 +276,18 @@ func (c ServerPreferredResources) OpenAPISchema() (*openapi_v2.Document, error)
}
// GetGVRFromKind get the Group Version Resource from kind
func (c ServerPreferredResources) GetGVRFromKind(kind string) schema.GroupVersionResource {
func (c ServerPreferredResources) GetGVRFromKind(kind string) (schema.GroupVersionResource, error) {
if kind == "" {
return schema.GroupVersionResource{}
return schema.GroupVersionResource{}, nil
}
_, gvr, err := c.FindResource("", kind)
if err != nil {
c.log.Info("schema not found", "kind", kind)
return schema.GroupVersionResource{}
return schema.GroupVersionResource{}, err
}
return gvr
return gvr, nil
}
// GetGVRFromAPIVersionKind get the Group Version Resource from APIVersion and kind

16
pkg/dclient/utils.go
View file

@ -39,8 +39,8 @@ func NewMockClient(scheme *runtime.Scheme, objects ...runtime.Object) (*Client,
}
// NewFakeDiscoveryClient returns a fakediscovery client
func NewFakeDiscoveryClient(registeredResouces []schema.GroupVersionResource) *fakeDiscoveryClient {
// Load some-preregistd resources
func NewFakeDiscoveryClient(registeredResources []schema.GroupVersionResource) *fakeDiscoveryClient {
// Load some-preregistered resources
res := []schema.GroupVersionResource{
{Version: "v1", Resource: "configmaps"},
{Version: "v1", Resource: "endpoints"},
@ -52,16 +52,16 @@ func NewFakeDiscoveryClient(registeredResouces []schema.GroupVersionResource) *f
{Group: "apps", Version: "v1", Resource: "deployments"},
{Group: "apps", Version: "v1", Resource: "statefulsets"},
}
registeredResouces = append(registeredResouces, res...)
return &fakeDiscoveryClient{registeredResouces: registeredResouces}
registeredResources = append(registeredResources, res...)
return &fakeDiscoveryClient{registeredResources: registeredResources}
}
type fakeDiscoveryClient struct {
registeredResouces []schema.GroupVersionResource
registeredResources []schema.GroupVersionResource
}
func (c *fakeDiscoveryClient) getGVR(resource string) schema.GroupVersionResource {
for _, gvr := range c.registeredResouces {
for _, gvr := range c.registeredResources {
if gvr.Resource == resource {
return gvr
}
@ -73,9 +73,9 @@ func (c *fakeDiscoveryClient) GetServerVersion() (*version.Info, error) {
return nil, nil
}
func (c *fakeDiscoveryClient) GetGVRFromKind(kind string) schema.GroupVersionResource {
func (c *fakeDiscoveryClient) GetGVRFromKind(kind string) (schema.GroupVersionResource, error) {
resource := strings.ToLower(kind) + "s"
return c.getGVR(resource)
return c.getGVR(resource), nil
}
func (c *fakeDiscoveryClient) GetGVRFromAPIVersionKind(apiVersion string, kind string) schema.GroupVersionResource {

11
pkg/generate/cleanup/controller.go
View file

@ -65,7 +65,7 @@ func NewController(
grInformer kyvernoinformer.GenerateRequestInformer,
dynamicInformer dynamicinformer.DynamicSharedInformerFactory,
log logr.Logger,
) *Controller {
) (*Controller, error) {
c := Controller{
kyvernoClient: kyvernoclient,
client: client,
@ -96,13 +96,18 @@ func NewController(
//TODO: dynamic registration
// Only supported for namespaces
nsInformer := dynamicInformer.ForResource(client.DiscoveryClient.GetGVRFromKind("Namespace"))
gvr, err := client.DiscoveryClient.GetGVRFromKind("Namespace")
if err != nil {
return nil, err
}
nsInformer := dynamicInformer.ForResource(gvr)
c.nsInformer = nsInformer
c.nsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
DeleteFunc: c.deleteGenericResource,
})
return &c
return &c, nil
}
func (c *Controller) deleteGenericResource(obj interface{}) {

13
pkg/generate/generate_controller.go
View file

@ -60,7 +60,7 @@ type Controller struct {
dynamicInformer dynamicinformer.DynamicSharedInformerFactory
//TODO: list of generic informers
// only support Namespaces for re-evalutation on resource updates
// only support Namespaces for re-evaluation on resource updates
nsInformer informers.GenericInformer
policyStatusListener policystatus.Listener
log logr.Logger
@ -81,7 +81,7 @@ func NewController(
log logr.Logger,
dynamicConfig config.Interface,
resourceCache resourcecache.ResourceCacheIface,
) *Controller {
) (*Controller, error) {
c := Controller{
client: client,
@ -116,13 +116,18 @@ func NewController(
//TODO: dynamic registration
// Only supported for namespaces
nsInformer := dynamicInformer.ForResource(client.DiscoveryClient.GetGVRFromKind("Namespace"))
gvr, err := client.DiscoveryClient.GetGVRFromKind("Namespace")
if err != nil {
return nil, err
}
nsInformer := dynamicInformer.ForResource(gvr)
c.nsInformer = nsInformer
c.nsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
UpdateFunc: c.updateGenericResource,
})
return &c
return &c, nil
}
func (c *Controller) updateGenericResource(old, cur interface{}) {

1
pkg/openapi/validation.go
View file

@ -326,4 +326,3 @@ func getAnyValue(any *openapi_v2.Any) []byte {
return nil
}

19
pkg/utils/util.go
View file

@ -5,6 +5,7 @@ import (
"reflect"
"regexp"
"strconv"
"strings"
engineutils "github.com/kyverno/kyverno/pkg/engine/utils"
"k8s.io/api/admission/v1beta1"
@ -61,7 +62,17 @@ func NewKubeClient(config *rest.Config) (kubernetes.Interface, error) {
func CRDInstalled(discovery client.IDiscovery, log logr.Logger) bool {
logger := log.WithName("CRDInstalled")
check := func(kind string) bool {
gvr := discovery.GetGVRFromKind(kind)
gvr, err := discovery.GetGVRFromKind(kind)
if err != nil {
if isServerUnavailable(err) {
logger.Info("**WARNING** unable to check CRD status", "kind", kind, "error", err.Error())
return true
}
logger.Error(err, "failed to check CRD status", "kind", kind)
return false
}
if reflect.DeepEqual(gvr, schema.GroupVersionResource{}) {
logger.Info("CRD not installed", "kind", kind)
return false
@ -183,3 +194,9 @@ func SliceContains(slice []string, values ...string) bool {
return false
}
func isServerUnavailable(err error) bool {
// error message -
// https://github.com/kubernetes/apimachinery/blob/2456ebdaba229616fab2161a615148884b46644b/pkg/api/errors/errors.go#L432
return strings.Contains(err.Error(), "the server is currently unable to handle the request")
}