From 586b197b0056b370594d3dc5080dc7a2b8590de7 Mon Sep 17 00:00:00 2001 From: Shuting Zhao Date: Mon, 11 Nov 2019 15:43:13 -0800 Subject: [PATCH] user sharedInformer for rolebindings and clusterrolebindings --- main.go | 3 +- pkg/userinfo/roleRef.go | 123 ++++++--------------- pkg/userinfo/roleRef_test.go | 203 +++++++++++++++++------------------ pkg/webhooks/server.go | 10 +- pkg/webhooks/utils.go | 5 +- 5 files changed, 145 insertions(+), 199 deletions(-) diff --git a/main.go b/main.go index 1f196ab23b..063501af9c 100644 --- a/main.go +++ b/main.go @@ -147,7 +147,8 @@ func main() { // -- annotations on resources with update details on mutation JSON patches // -- generate policy violation resource // -- generate events on policy and resource - server, err := webhooks.NewWebhookServer(pclient, client, tlsPair, pInformer.Kyverno().V1alpha1().ClusterPolicies(), pInformer.Kyverno().V1alpha1().ClusterPolicyViolations(), egen, webhookRegistrationClient, pc.GetPolicyStatusAggregator(), configData, cleanUp) + server, err := webhooks.NewWebhookServer(pclient, client, tlsPair, pInformer.Kyverno().V1alpha1().ClusterPolicies(), pInformer.Kyverno().V1alpha1().ClusterPolicyViolations(), + kubeInformer.Rbac().V1().RoleBindings(),kubeInformer.Rbac().V1().ClusterRoleBindings(),egen, webhookRegistrationClient, pc.GetPolicyStatusAggregator(), configData, cleanUp) if err != nil { glog.Fatalf("Unable to create webhook server: %v\n", err) } diff --git a/pkg/userinfo/roleRef.go b/pkg/userinfo/roleRef.go index 3855129537..12a43c6f6b 100644 --- a/pkg/userinfo/roleRef.go +++ b/pkg/userinfo/roleRef.go @@ -5,42 +5,35 @@ import ( "strings" "github.com/golang/glog" - client "github.com/nirmata/kyverno/pkg/dclient" engine "github.com/nirmata/kyverno/pkg/engine" v1beta1 "k8s.io/api/admission/v1beta1" authenticationv1 "k8s.io/api/authentication/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + rbacv1 "k8s.io/api/rbac/v1" + labels "k8s.io/apimachinery/pkg/labels" + rbaclister "k8s.io/client-go/listers/rbac/v1" ) -func GetRoleRef(client *client.Client, request *v1beta1.AdmissionRequest) (roles []string, clusterRoles []string, err error) { - nsList, err := client.ListResource("Namespace", "", metav1.ListOptions{}) - if err != nil { - return nil, nil, fmt.Errorf("failed to get namespace list: %v", err) - } - +func GetRoleRef(rbLister rbaclister.RoleBindingLister, crbLister rbaclister.ClusterRoleBindingLister, request *v1beta1.AdmissionRequest) (roles []string, clusterRoles []string, err error) { // rolebindings - for _, ns := range nsList.Items { - roleBindings, err := client.ListResource("RoleBindings", ns.GetName(), metav1.ListOptions{}) - if err != nil { - return roles, clusterRoles, fmt.Errorf("failed to list rolebindings: %v", err) - } - - rs, crs, err := getRoleRefByRoleBindings(roleBindings, request.UserInfo) - if err != nil { - return roles, clusterRoles, err - } - roles = append(roles, rs...) - clusterRoles = append(clusterRoles, crs...) + roleBindings, err := rbLister.List(labels.NewSelector()) + if err != nil { + return roles, clusterRoles, fmt.Errorf("failed to list rolebindings: %v", err) } + rs, crs, err := getRoleRefByRoleBindings(roleBindings, request.UserInfo) + if err != nil { + return roles, clusterRoles, err + } + roles = append(roles, rs...) + clusterRoles = append(clusterRoles, crs...) + // clusterrolebindings - clusterroleBindings, err := client.ListResource("ClusterRoleBindings", "", metav1.ListOptions{}) + clusterroleBindings, err := crbLister.List(labels.NewSelector()) if err != nil { return roles, clusterRoles, fmt.Errorf("failed to list clusterrolebindings: %v", err) } - crs, err := getRoleRefByClusterRoleBindings(clusterroleBindings, request.UserInfo) + crs, err = getRoleRefByClusterRoleBindings(clusterroleBindings, request.UserInfo) if err != nil { return roles, clusterRoles, err } @@ -49,30 +42,19 @@ func GetRoleRef(client *client.Client, request *v1beta1.AdmissionRequest) (roles return roles, clusterRoles, nil } -func getRoleRefByRoleBindings(roleBindings *unstructured.UnstructuredList, userInfo authenticationv1.UserInfo) (roles []string, clusterRoles []string, err error) { - for _, rolebinding := range roleBindings.Items { - rb := rolebinding.UnstructuredContent() - subjects, ok := rb["subjects"] - if !ok { - return nil, nil, fmt.Errorf("%s/%s/%s has no subjects field", rolebinding.GetKind(), rolebinding.GetNamespace(), rolebinding.GetName()) - } - - roleRef, ok := rb["roleRef"] - if !ok { - return nil, nil, fmt.Errorf("%s/%s/%s has no roleRef", rolebinding.GetKind(), rolebinding.GetNamespace(), rolebinding.GetName()) - } - - for _, subject := range subjects.([]map[string]interface{}) { +func getRoleRefByRoleBindings(roleBindings []*rbacv1.RoleBinding, userInfo authenticationv1.UserInfo) (roles []string, clusterRoles []string, err error) { + for _, rolebinding := range roleBindings { + for _, subject := range rolebinding.Subjects { if !matchSubjectsMap(subject, userInfo) { continue } - roleRefMap := roleRef.(map[string]interface{}) - switch roleRefMap["kind"] { + // roleRefMap := roleRef.(map[string]interface{}) + switch rolebinding.RoleRef.Kind { case "role": - roles = append(roles, roleRefMap["namespace"].(string)+":"+roleRefMap["name"].(string)) + roles = append(roles, rolebinding.Namespace+":"+rolebinding.RoleRef.Name) case "clusterRole": - clusterRoles = append(clusterRoles, roleRefMap["name"].(string)) + clusterRoles = append(clusterRoles, rolebinding.RoleRef.Name) } } } @@ -81,27 +63,15 @@ func getRoleRefByRoleBindings(roleBindings *unstructured.UnstructuredList, userI } // RoleRef in ClusterRoleBindings can only reference a ClusterRole in the global namespace -func getRoleRefByClusterRoleBindings(clusterroleBindings *unstructured.UnstructuredList, userInfo authenticationv1.UserInfo) (clusterRoles []string, err error) { - for _, clusterRoleBinding := range clusterroleBindings.Items { - crb := clusterRoleBinding.UnstructuredContent() - subjects, ok := crb["subjects"] - if !ok { - return nil, fmt.Errorf("%s/%s has no subjects field", clusterRoleBinding.GetKind(), clusterRoleBinding.GetName()) - } - - roleRef, ok := crb["roleRef"] - if !ok { - return nil, fmt.Errorf("%s/%s has no roleRef", clusterRoleBinding.GetKind(), clusterRoleBinding.GetName()) - } - - for _, subject := range subjects.([]map[string]interface{}) { +func getRoleRefByClusterRoleBindings(clusterroleBindings []*rbacv1.ClusterRoleBinding, userInfo authenticationv1.UserInfo) (clusterRoles []string, err error) { + for _, clusterRoleBinding := range clusterroleBindings { + for _, subject := range clusterRoleBinding.Subjects { if !matchSubjectsMap(subject, userInfo) { continue } - roleRefMap := roleRef.(map[string]interface{}) - if roleRefMap["kind"] == "clusterRole" { - clusterRoles = append(clusterRoles, roleRefMap["name"].(string)) + if clusterRoleBinding.RoleRef.Kind == "clusterRole" { + clusterRoles = append(clusterRoles, clusterRoleBinding.RoleRef.Name) } } } @@ -111,7 +81,7 @@ func getRoleRefByClusterRoleBindings(clusterroleBindings *unstructured.Unstructu // matchSubjectsMap checks if userInfo found in subject // return true directly if found a match // subject["kind"] can only be ServiceAccount, User and Group -func matchSubjectsMap(subject map[string]interface{}, userInfo authenticationv1.UserInfo) bool { +func matchSubjectsMap(subject rbacv1.Subject, userInfo authenticationv1.UserInfo) bool { // ServiceAccount if isServiceaccountUserInfo(userInfo.Username) { return matchServiceAccount(subject, userInfo) @@ -130,33 +100,8 @@ func isServiceaccountUserInfo(username string) bool { // matchServiceAccount checks if userInfo sa matche the subject sa // serviceaccount represents as saPrefix:namespace:name in userInfo -func matchServiceAccount(subject map[string]interface{}, userInfo authenticationv1.UserInfo) bool { - // checks if subject contains the serviceaccount info - sa, ok := subject["kind"].(string) - if !ok { - glog.V(3).Infof("subject %v has wrong kind field", subject) - return false - } - - if sa != "ServiceAccount" { - glog.V(3).Infof("subject %v has no ServiceAccount info", subject) - return false - } - - namespace, ok := subject["namespace"].(string) - if !ok { - glog.V(3).Infof("subject %v has wrong namespace field", subject) - return false - } - - _ = subject["name"] - name, ok := subject["name"].(string) - if !ok { - glog.V(3).Infof("subject %v has wrong name field", subject) - return false - } - - subjectServiceAccount := namespace + ":" + name +func matchServiceAccount(subject rbacv1.Subject, userInfo authenticationv1.UserInfo) bool { + subjectServiceAccount := subject.Namespace + ":" + subject.Name if userInfo.Username[len(engine.SaPrefix):] != subjectServiceAccount { glog.V(3).Infof("service account not match, expect %s, got %s", subjectServiceAccount, userInfo.Username[len(engine.SaPrefix):]) return false @@ -166,14 +111,14 @@ func matchServiceAccount(subject map[string]interface{}, userInfo authentication } // matchUserOrGroup checks if userInfo contains user or group info in a subject -func matchUserOrGroup(subject map[string]interface{}, userInfo authenticationv1.UserInfo) bool { +func matchUserOrGroup(subject rbacv1.Subject, userInfo authenticationv1.UserInfo) bool { keys := append(userInfo.Groups, userInfo.Username) for _, key := range keys { - if subject["name"].(string) == key { + if subject.Name == key { return true } } - glog.V(3).Infof("user/group '%v' info not found in request userInfo: %v", subject["name"], keys) + glog.V(3).Infof("user/group '%v' info not found in request userInfo: %v", subject.Name, keys) return false } diff --git a/pkg/userinfo/roleRef_test.go b/pkg/userinfo/roleRef_test.go index 6562778d04..cca7864209 100644 --- a/pkg/userinfo/roleRef_test.go +++ b/pkg/userinfo/roleRef_test.go @@ -7,7 +7,8 @@ import ( "gotest.tools/assert" authenticationv1 "k8s.io/api/authentication/v1" - unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func Test_isServiceaccountUserInfo(t *testing.T) { @@ -37,46 +38,46 @@ func Test_matchServiceAccount_subject_variants(t *testing.T) { } tests := []struct { - subject map[string]interface{} + subject rbacv1.Subject expected bool }{ { - subject: make(map[string]interface{}, 1), + subject: rbacv1.Subject{}, expected: false, }, { - subject: map[string]interface{}{ - "kind": "serviceaccount", + subject: rbacv1.Subject{ + Kind: "serviceaccount", }, expected: false, }, { - subject: map[string]interface{}{ - "kind": "ServiceAccount", - "Namespace": "testnamespace", + subject: rbacv1.Subject{ + Kind: "ServiceAccount", + Namespace: "testnamespace", }, expected: false, }, { - subject: map[string]interface{}{ - "kind": "ServiceAccount", - "namespace": 1, + subject: rbacv1.Subject{ + Kind: "ServiceAccount", + Namespace: "1", }, expected: false, }, { - subject: map[string]interface{}{ - "kind": "ServiceAccount", - "namespace": "testnamespace", - "names": "", + subject: rbacv1.Subject{ + Kind: "ServiceAccount", + Namespace: "testnamespace", + Name: "", }, expected: false, }, { - subject: map[string]interface{}{ - "kind": "ServiceAccount", - "namespace": "testnamespace", - "name": "testname", + subject: rbacv1.Subject{ + Kind: "ServiceAccount", + Namespace: "testnamespace", + Name: "testname", }, expected: false, }, @@ -104,19 +105,19 @@ func Test_matchUserOrGroup(t *testing.T) { Groups: []string{"system:authenticated"}, } - userContext := map[string]interface{}{ - "kind": "User", - "name": "system:kube-scheduler", + userContext := rbacv1.Subject{ + Kind: "User", + Name: "system:kube-scheduler", } - groupContext := map[string]interface{}{ - "kind": "Group", - "name": "system:masters", + groupContext := rbacv1.Subject{ + Kind: "Group", + Name: "system:masters", } - fakegroupContext := map[string]interface{}{ - "kind": "Group", - "name": "fakeGroup", + fakegroupContext := rbacv1.Subject{ + Kind: "Group", + Name: "fakeGroup", } res := matchUserOrGroup(userContext, user) @@ -142,15 +143,15 @@ func Test_matchSubjectsMap(t *testing.T) { Groups: []string{"system:masters", "system:authenticated"}, } - sasubject := map[string]interface{}{ - "kind": "ServiceAccount", - "namespace": "default", - "name": "saconfig", + sasubject := rbacv1.Subject{ + Kind: "ServiceAccount", + Namespace: "default", + Name: "saconfig", } - groupsubject := map[string]interface{}{ - "kind": "Group", - "name": "fakeGroup", + groupsubject := rbacv1.Subject{ + Kind: "Group", + Name: "fakeGroup", } res := matchSubjectsMap(sasubject, sa) @@ -164,46 +165,43 @@ func Test_getRoleRefByRoleBindings(t *testing.T) { flag.Parse() flag.Set("logtostderr", "true") flag.Set("v", "3") - list := &unstructured.UnstructuredList{ - Object: map[string]interface{}{"kind": "List", "apiVersion": "v1"}, - Items: []unstructured.Unstructured{ - { - Object: map[string]interface{}{ - "kind": "RoleBinding", - "apiVersion": "rbac.authorization.k8s.io/v1", - "metadata": map[string]interface{}{"name": "test1"}, - "roleRef": map[string]interface{}{ - "kind": "role", - "name": "myrole", - "namespace": "mynamespace", - }, - "subjects": []map[string]interface{}{ - { - "kind": "ServiceAccount", - "name": "saconfig", - "namespace": "default", - }, - }, + + list := []*rbacv1.RoleBinding{ + &rbacv1.RoleBinding{ + metav1.TypeMeta{ + Kind: "RoleBinding", + APIVersion: "rbac.authorization.k8s.io/v1", + }, + metav1.ObjectMeta{Name: "test1", Namespace: "mynamespace"}, + []rbacv1.Subject{ + rbacv1.Subject{ + Kind: "ServiceAccount", + Name: "saconfig", + Namespace: "default", }, }, - { - Object: map[string]interface{}{ - "kind": "RoleBinding", - "apiVersion": "rbac.authorization.k8s.io/v1", - "metadata": map[string]interface{}{"name": "test2"}, - "roleRef": map[string]interface{}{ - "kind": "clusterRole", - "name": "myclusterrole", - }, - "subjects": []map[string]interface{}{ - { - "kind": "ServiceAccount", - "name": "saconfig", - "namespace": "default", - }, - }, + rbacv1.RoleRef{ + Kind: "role", + Name: "myrole", + }, + }, + &rbacv1.RoleBinding{ + metav1.TypeMeta{ + Kind: "RoleBinding", + APIVersion: "rbac.authorization.k8s.io/v1", + }, + metav1.ObjectMeta{Name: "test2", Namespace: "mynamespace"}, + []rbacv1.Subject{ + rbacv1.Subject{ + Kind: "ServiceAccount", + Name: "saconfig", + Namespace: "default", }, }, + rbacv1.RoleRef{ + Kind: "clusterRole", + Name: "myclusterrole", + }, }, } @@ -220,43 +218,40 @@ func Test_getRoleRefByRoleBindings(t *testing.T) { } func Test_getRoleRefByClusterRoleBindings(t *testing.T) { - list := &unstructured.UnstructuredList{ - Object: map[string]interface{}{"kind": "List", "apiVersion": "v1"}, - Items: []unstructured.Unstructured{ - { - Object: map[string]interface{}{ - "kind": "ClusterRoleBinding", - "apiVersion": "rbac.authorization.k8s.io/v1", - "metadata": map[string]interface{}{"name": "test-1"}, - "roleRef": map[string]interface{}{ - "kind": "clusterRole", - "name": "fakeclusterrole", - }, - "subjects": []map[string]interface{}{ - { - "kind": "User", - "name": "kube-scheduler", - }, - }, + list := []*rbacv1.ClusterRoleBinding{ + &rbacv1.ClusterRoleBinding{ + metav1.TypeMeta{ + Kind: "ClusterRoleBinding", + APIVersion: "rbac.authorization.k8s.io/v1", + }, + metav1.ObjectMeta{Name: "test1", Namespace: "mynamespace"}, + []rbacv1.Subject{ + rbacv1.Subject{ + Kind: "User", + Name: "kube-scheduler", }, }, - { - Object: map[string]interface{}{ - "kind": "ClusterRoleBinding", - "apiVersion": "rbac.authorization.k8s.io/v1", - "metadata": map[string]interface{}{"name": "test-2"}, - "roleRef": map[string]interface{}{ - "kind": "clusterRole", - "name": "myclusterrole", - }, - "subjects": []map[string]interface{}{ - { - "kind": "Group", - "name": "system:masters", - }, - }, + rbacv1.RoleRef{ + Kind: "clusterRole", + Name: "fakeclusterrole", + }, + }, + &rbacv1.ClusterRoleBinding{ + metav1.TypeMeta{ + Kind: "ClusterRoleBinding", + APIVersion: "rbac.authorization.k8s.io/v1", + }, + metav1.ObjectMeta{Name: "test2", Namespace: "mynamespace"}, + []rbacv1.Subject{ + rbacv1.Subject{ + Kind: "Group", + Name: "system:masters", }, }, + rbacv1.RoleRef{ + Kind: "clusterRole", + Name: "myclusterrole", + }, }, } diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index 8550c03841..82e01c4de9 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -24,6 +24,8 @@ import ( v1beta1 "k8s.io/api/admission/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + rbacinformer "k8s.io/client-go/informers/rbac/v1" + rbaclister "k8s.io/client-go/listers/rbac/v1" "k8s.io/client-go/tools/cache" ) @@ -37,6 +39,8 @@ type WebhookServer struct { pvLister kyvernolister.ClusterPolicyViolationLister pListerSynced cache.InformerSynced pvListerSynced cache.InformerSynced + rbLister rbaclister.RoleBindingLister + crbLister rbaclister.ClusterRoleBindingLister eventGen event.Interface webhookRegistrationClient *webhookconfig.WebhookRegistrationClient // API to send policy stats for aggregation @@ -55,6 +59,8 @@ func NewWebhookServer( tlsPair *tlsutils.TlsPemPair, pInformer kyvernoinformer.ClusterPolicyInformer, pvInformer kyvernoinformer.ClusterPolicyViolationInformer, + rbInformer rbacinformer.RoleBindingInformer, + crbInformer rbacinformer.ClusterRoleBindingInformer, eventGen event.Interface, webhookRegistrationClient *webhookconfig.WebhookRegistrationClient, policyStatus policy.PolicyStatusInterface, @@ -84,6 +90,8 @@ func NewWebhookServer( webhookRegistrationClient: webhookRegistrationClient, policyStatus: policyStatus, configHandler: configHandler, + rbLister: rbInformer.Lister(), + crbLister: crbInformer.Lister(), cleanUp: cleanUp, } mux := http.NewServeMux() @@ -158,7 +166,7 @@ func (ws *WebhookServer) handleAdmissionRequest(request *v1beta1.AdmissionReques // TODO(shuting): replace containRBACinfo after policy cache lookup is introduced // getRoleRef only if policy has roles/clusterroles defined if containRBACinfo(policies) { - roles, clusterRoles, err = userinfo.GetRoleRef(ws.client, request) + roles, clusterRoles, err = userinfo.GetRoleRef(ws.rbLister, ws.crbLister, request) if err != nil { // TODO(shuting): continue apply policy if error getting roleRef? glog.Errorf("Unable to get rbac information for request Kind=%s, Namespace=%s Name=%s UID=%s patchOperation=%s: %v", diff --git a/pkg/webhooks/utils.go b/pkg/webhooks/utils.go index 9d2f387e5a..7dcfbb5b04 100644 --- a/pkg/webhooks/utils.go +++ b/pkg/webhooks/utils.go @@ -98,10 +98,7 @@ func processResourceWithPatches(patch []byte, resource []byte) []byte { func containRBACinfo(policies []*kyverno.ClusterPolicy) bool { for _, policy := range policies { for _, rule := range policy.Spec.Rules { - if len(rule.MatchResources.Roles) > 0 { - return true - } - if len(rule.MatchResources.ClusterRoles) > 0 { + if len(rule.MatchResources.Roles) > 0 || len(rule.MatchResources.ClusterRoles) > 0 { return true } }