feat: add e2e tests for aws role-based auth (#2376)

Signed-off-by: Moritz Johner <beller.moritz@googlemail.com>

cmd/root.go

@@ -156,7 +156,9 @@ var rootCmd = &cobra.Command{
 			Scheme:          mgr.GetScheme(),
 			ControllerClass: controllerClass,
 			RequeueInterval: storeRequeueInterval,
-		}).SetupWithManager(mgr); err != nil {
+		}).SetupWithManager(mgr, controller.Options{
+			MaxConcurrentReconciles: concurrent,
+		}); err != nil {
 			setupLog.Error(err, errCreateController, "controller", "SecretStore")
 			os.Exit(1)
 		}
@@ -242,7 +244,7 @@ func init() {
 	rootCmd.Flags().BoolVar(&enableLeaderElection, "enable-leader-election", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
-	rootCmd.Flags().IntVar(&concurrent, "concurrent", 1, "The number of concurrent ExternalSecret reconciles.")
+	rootCmd.Flags().IntVar(&concurrent, "concurrent", 1, "The number of concurrent reconciles.")
 	rootCmd.Flags().Float32Var(&clientQPS, "client-qps", 0, "QPS configuration to be passed to rest.Client")
 	rootCmd.Flags().IntVar(&clientBurst, "client-burst", 0, "Maximum Burst allowed to be passed to rest.Client")
 	rootCmd.Flags().StringVar(&loglevel, "loglevel", "info", "loglevel to use, one of: debug, info, warn, error, dpanic, panic, fatal")

docs/api/controller-options.md

@@ -11,11 +11,11 @@ The external-secrets binary includes three components: `core controller`, `certc
 
 The core controller is invoked without a subcommand and can be configured with the following flags:
 
-| Name                                          | Type     | Default                       | Description                                                                                                                                                         |
+| Name                                          | Type     | Default                       | Description                                                                                                                                                        |
 | --------------------------------------------- | -------- | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
 | `--client-burst`                              | int      | uses rest client default (10) | Maximum Burst allowed to be passed to rest.Client                                                                                                                  |
 | `--client-qps`                                | float32  | uses rest client default (5)  | QPS configuration to be passed to rest.Client                                                                                                                      |
-| `--concurrent`                                | int      | 1                             | The number of concurrent ExternalSecret reconciles.                                                                                                                |
+| `--concurrent`                                | int      | 1                             | The number of concurrent reconciles.                                                                                                                               |
 | `--controller-class`                          | string   | default                       | The controller is instantiated with a specific controller name and filters ES based on this property                                                               |
 | `--enable-cluster-external-secret-reconciler` | boolean  | true                          | Enables the cluster external secret reconciler.                                                                                                                    |
 | `--enable-cluster-store-reconciler`           | boolean  | true                          | Enables the cluster store reconciler.                                                                                                                              |
@@ -23,7 +23,7 @@ The core controller is invoked without a subcommand and can be configured with t
 | `--enable-secrets-caching`                    | boolean  | false                         | Enables the secrets caching for external-secrets pod.                                                                                                              |
 | `--enable-configmaps-caching`                 | boolean  | false                         | Enables the ConfigMap caching for external-secrets pod.                                                                                                            |
 | `--enable-flood-gate`                         | boolean  | true                          | Enable flood gate. External secret will be reconciled only if the ClusterStore or Store have an healthy or unknown state.                                          |
-| `--enable-extended-metric-labels`             | boolean  | true                          | Enable recommended kubernetes annotations as labels in metrics.                                          |
+| `--enable-extended-metric-labels`             | boolean  | true                          | Enable recommended kubernetes annotations as labels in metrics.                                                                                                    |
 | `--enable-leader-election`                    | boolean  | false                         | Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.                                              |
 | `--experimental-enable-aws-session-cache`     | boolean  | false                         | Enable experimental AWS session cache. External secret will reuse the AWS session without creating a new one on each request.                                      |
 | `--help`                                      |          |                               | help for external-secrets                                                                                                                                          |
@@ -49,7 +49,7 @@ The core controller is invoked without a subcommand and can be configured with t
 
 ## Webhook Flags
 
-| Name                   | Type     | Default                               | Description                                                                                                                                                                                                                                                                                                                                                                                                               |
+| Name                   | Type     | Default                               | Description                                                                                                                                                                                                                                                                                                                                                                                                              |
 | ---------------------- | -------- | ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
 | `--cert-dir`           | string   | /tmp/k8s-webhook-server/serving-certs | path to check for certs                                                                                                                                                                                                                                                                                                                                                                                                  |
 | `--check-interval`     | duration | 5m0s                                  | certificate check interval                                                                                                                                                                                                                                                                                                                                                                                               |

e2e/run.sh

@@ -45,6 +45,7 @@ kubectl run --rm \
   --restart=Never \
   --pod-running-timeout=5m \
   --labels="app=eso-e2e" \
+  --env="ACK_GINKGO_DEPRECATIONS=2.9.5" \
   --env="GINKGO_LABELS=${GINKGO_LABELS:-.*}" \
   --env="GCP_SM_SA_JSON=${GCP_SM_SA_JSON:-}" \
   --env="GCP_PROJECT_ID=${GCP_PROJECT_ID:-}" \

e2e/suites/provider/cases/aws/common.go

@@ -31,6 +31,11 @@ const (
 	WithMountedIRSA                     = "with mounted IRSA"
 	StaticCredentialsSecretName         = "provider-secret"
 	StaticReferentCredentialsSecretName = "referent-provider-secret"
+
+	IAMRoleExternalID  = "arn:aws:iam::783882199045:role/eso-e2e-external-id"
+	IAMRoleSessionTags = "arn:aws:iam::783882199045:role/eso-e2e-session-tags"
+
+	IAMTrustedExternalID = "eso-e2e-ext-id"
 )
 
 func ReferencedIRSAStoreName(f *framework.Framework) string {
@@ -53,28 +58,33 @@ func UseMountedIRSAStore(tc *framework.TestCase) {
 
 const (
 	StaticStoreName       = "aws-static-creds"
+	ExternalIDStoreName   = "aws-ext-id"
+	SessionTagsStoreName  = "aws-sess-tags"
 	staticKeyID           = "kid"
 	staticSecretAccessKey = "sak"
 	staticySessionToken   = "st"
 )
 
-func newStaticStoreProvider(serviceType esv1beta1.AWSServiceType, region, secretName string) *esv1beta1.SecretStoreProvider {
+func newStaticStoreProvider(serviceType esv1beta1.AWSServiceType, region, secretName, role, externalID string, sessionTags []*esv1beta1.Tag) *esv1beta1.SecretStoreProvider {
 	return &esv1beta1.SecretStoreProvider{
 		AWS: &esv1beta1.AWSProvider{
-			Service: serviceType,
+			Service:     serviceType,
-			Region:  region,
+			Region:      region,
+			Role:        role,
+			ExternalID:  externalID,
+			SessionTags: sessionTags,
 			Auth: esv1beta1.AWSAuth{
 				SecretRef: &esv1beta1.AWSAuthSecretRef{
 					AccessKeyID: esmetav1.SecretKeySelector{
-						Name: StaticReferentCredentialsSecretName,
+						Name: secretName,
 						Key:  staticKeyID,
 					},
 					SecretAccessKey: esmetav1.SecretKeySelector{
-						Name: StaticReferentCredentialsSecretName,
+						Name: secretName,
 						Key:  staticSecretAccessKey,
 					},
 					SessionToken: &esmetav1.SecretKeySelector{
-						Name: StaticReferentCredentialsSecretName,
+						Name: secretName,
 						Key:  staticySessionToken,
 					},
 				},
@@ -83,6 +93,68 @@ func newStaticStoreProvider(serviceType esv1beta1.AWSServiceType, region, secret
 	}
 }
 
+// SessionTagsStore is namespaced and references
+// static credentials from a secret. It assumes a role and specifies session tags
+func SetupSessionTagsStore(f *framework.Framework, kid, sak, st, region, role string, sessionTags []*esv1beta1.Tag, serviceType esv1beta1.AWSServiceType) {
+	credsName := "provider-secret-sess-tags"
+	awsCreds := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      credsName,
+			Namespace: f.Namespace.Name,
+		},
+		StringData: map[string]string{
+			staticKeyID:           kid,
+			staticSecretAccessKey: sak,
+			staticySessionToken:   st,
+		},
+	}
+	err := f.CRClient.Create(context.Background(), awsCreds)
+	Expect(err).ToNot(HaveOccurred())
+
+	secretStore := &esv1beta1.SecretStore{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      SessionTagsStoreName,
+			Namespace: f.Namespace.Name,
+		},
+		Spec: esv1beta1.SecretStoreSpec{
+			Provider: newStaticStoreProvider(serviceType, region, credsName, role, "", sessionTags),
+		},
+	}
+	err = f.CRClient.Create(context.Background(), secretStore)
+	Expect(err).ToNot(HaveOccurred())
+}
+
+// ExternalIDStore is namespaced and references
+// static credentials from a secret. It assumes a role and specifies an externalID
+func SetupExternalIDStore(f *framework.Framework, kid, sak, st, region, role, externalID string, sessionTags []*esv1beta1.Tag, serviceType esv1beta1.AWSServiceType) {
+	credsName := "provider-secret-ext-id"
+	awsCreds := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      credsName,
+			Namespace: f.Namespace.Name,
+		},
+		StringData: map[string]string{
+			staticKeyID:           kid,
+			staticSecretAccessKey: sak,
+			staticySessionToken:   st,
+		},
+	}
+	err := f.CRClient.Create(context.Background(), awsCreds)
+	Expect(err).ToNot(HaveOccurred())
+
+	secretStore := &esv1beta1.SecretStore{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      ExternalIDStoreName,
+			Namespace: f.Namespace.Name,
+		},
+		Spec: esv1beta1.SecretStoreSpec{
+			Provider: newStaticStoreProvider(serviceType, region, credsName, role, externalID, sessionTags),
+		},
+	}
+	err = f.CRClient.Create(context.Background(), secretStore)
+	Expect(err).ToNot(HaveOccurred())
+}
+
 // StaticStore is namespaced and references
 // static credentials from a secret.
 func SetupStaticStore(f *framework.Framework, kid, sak, st, region string, serviceType esv1beta1.AWSServiceType) {
@@ -106,7 +178,7 @@ func SetupStaticStore(f *framework.Framework, kid, sak, st, region string, servi
 			Namespace: f.Namespace.Name,
 		},
 		Spec: esv1beta1.SecretStoreSpec{
-			Provider: newStaticStoreProvider(serviceType, region, StaticCredentialsSecretName),
+			Provider: newStaticStoreProvider(serviceType, region, StaticCredentialsSecretName, "", "", nil),
 		},
 	}
 	err = f.CRClient.Create(context.Background(), secretStore)
@@ -137,7 +209,7 @@ func CreateReferentStaticStore(f *framework.Framework, kid, sak, st, region stri
 			Name: ReferentSecretStoreName(f),
 		},
 		Spec: esv1beta1.SecretStoreSpec{
-			Provider: newStaticStoreProvider(serviceType, region, StaticReferentCredentialsSecretName),
+			Provider: newStaticStoreProvider(serviceType, region, StaticReferentCredentialsSecretName, "", "", nil),
 		},
 	}
 	err = f.CRClient.Create(context.Background(), secretStore)

e2e/suites/provider/cases/aws/secretsmanager/assume_role.go

@@ -0,0 +1,61 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+limitations under the License.
+*/
+package aws
+
+import (
+	"fmt"
+
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/external-secrets/external-secrets-e2e/framework"
+	esapi "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
+)
+
+// This case creates secret with specific tags which are checked by the assumed IAM policy
+func SimpleSyncWithNamespaceTags(prov *Provider) func(f *framework.Framework) (string, func(*framework.TestCase)) {
+	return func(f *framework.Framework) (string, func(*framework.TestCase)) {
+		return "[common] should sync tagged simple secrets from .Data[]", func(tc *framework.TestCase) {
+			secretKey1 := fmt.Sprintf("%s-%s", f.Namespace.Name, "one")
+			secretKey2 := fmt.Sprintf("%s-%s", f.Namespace.Name, "other")
+			remoteRefKey1 := f.MakeRemoteRefKey(secretKey1)
+			remoteRefKey2 := f.MakeRemoteRefKey(secretKey2)
+			secretValue := "bar"
+			tc.Secrets = map[string]framework.SecretEntry{
+				// add specific tags to the secret resource. The assumed role only allows access to those
+				remoteRefKey1: {Value: secretValue, Tags: map[string]string{"namespace": "e2e-test"}},
+				remoteRefKey2: {Value: secretValue, Tags: map[string]string{"namespace": "e2e-test"}},
+			}
+			tc.ExpectedSecret = &v1.Secret{
+				Type: v1.SecretTypeOpaque,
+				Data: map[string][]byte{
+					secretKey1: []byte(secretValue),
+					secretKey2: []byte(secretValue),
+				},
+			}
+			tc.ExternalSecret.Spec.Data = []esapi.ExternalSecretData{
+				{
+					SecretKey: secretKey1,
+					RemoteRef: esapi.ExternalSecretDataRemoteRef{
+						Key: remoteRefKey1,
+					},
+				},
+				{
+					SecretKey: secretKey2,
+					RemoteRef: esapi.ExternalSecretDataRemoteRef{
+						Key: remoteRefKey2,
+					},
+				},
+			}
+		}
+	}
+}

e2e/suites/provider/cases/aws/secretsmanager/provider.go

@@ -70,6 +70,8 @@ func NewProvider(f *framework.Framework, kid, sak, st, region, saName, saNamespa
 
 	BeforeEach(func() {
 		awscommon.SetupStaticStore(f, kid, sak, st, region, esv1beta1.AWSServiceSecretsManager)
+		awscommon.SetupExternalIDStore(f, kid, sak, st, region, awscommon.IAMRoleExternalID, awscommon.IAMTrustedExternalID, nil, esv1beta1.AWSServiceSecretsManager)
+		awscommon.SetupSessionTagsStore(f, kid, sak, st, region, awscommon.IAMRoleSessionTags, nil, esv1beta1.AWSServiceSecretsManager)
 		awscommon.CreateReferentStaticStore(f, kid, sak, st, region, esv1beta1.AWSServiceSecretsManager)
 		prov.SetupReferencedIRSAStore()
 		prov.SetupMountedIRSAStore()
@@ -78,7 +80,6 @@ func NewProvider(f *framework.Framework, kid, sak, st, region, saName, saNamespa
 	AfterEach(func() {
 		prov.TeardownReferencedIRSAStore()
 		prov.TeardownMountedIRSAStore()
-
 	})
 
 	return prov

e2e/suites/provider/cases/aws/secretsmanager/secretsmanager.go

@@ -27,6 +27,8 @@ import (
 
 const (
 	withStaticAuth         = "with static auth"
+	withExtID              = "with externalID"
+	withSessionTags        = "with session tags"
 	withReferentStaticAuth = "with static referent auth"
 )
 
@@ -58,6 +60,10 @@ var _ = Describe("[aws] ", Label("aws", "secretsmanager"), func() {
 
 		// referent auth
 		framework.Compose(withStaticAuth, f, common.SimpleDataSync, useReferentStaticAuth),
+
+		// test assume role with external-id and session tags
+		framework.Compose(withExtID, f, SimpleSyncWithNamespaceTags(prov), useExtIDAuth),
+		framework.Compose(withSessionTags, f, SimpleSyncWithNamespaceTags(prov), useSessionTagsAuth),
 	)
 })
 
@@ -68,6 +74,20 @@ func useStaticAuth(tc *framework.TestCase) {
 	}
 }
 
+func useExtIDAuth(tc *framework.TestCase) {
+	tc.ExternalSecret.Spec.SecretStoreRef.Name = awscommon.ExternalIDStoreName
+	if tc.ExternalSecretV1Alpha1 != nil {
+		tc.ExternalSecretV1Alpha1.Spec.SecretStoreRef.Name = awscommon.ExternalIDStoreName
+	}
+}
+
+func useSessionTagsAuth(tc *framework.TestCase) {
+	tc.ExternalSecret.Spec.SecretStoreRef.Name = awscommon.SessionTagsStoreName
+	if tc.ExternalSecretV1Alpha1 != nil {
+		tc.ExternalSecretV1Alpha1.Spec.SecretStoreRef.Name = awscommon.SessionTagsStoreName
+	}
+}
+
 func useReferentStaticAuth(tc *framework.TestCase) {
 	tc.ExternalSecret.Spec.SecretStoreRef.Name = awscommon.ReferentSecretStoreName(tc.Framework)
 	tc.ExternalSecret.Spec.SecretStoreRef.Kind = esapi.ClusterSecretStoreKind

pkg/controllers/secretstore/secretstore_controller.go

@@ -24,6 +24,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 
 	esapi "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
 	ctrlmetrics "github.com/external-secrets/external-secrets/pkg/controllers/metrics"
@@ -64,10 +65,11 @@ func (r *StoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl
 }
 
 // SetupWithManager returns a new controller builder that will be started by the provided Manager.
-func (r *StoreReconciler) SetupWithManager(mgr ctrl.Manager) error {
+func (r *StoreReconciler) SetupWithManager(mgr ctrl.Manager, opts controller.Options) error {
 	r.recorder = mgr.GetEventRecorderFor("secret-store")
 
 	return ctrl.NewControllerManagedBy(mgr).
+		WithOptions(opts).
 		For(&esapi.SecretStore{}).
 		Complete(r)
 }

pkg/controllers/secretstore/suite_test.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/client-go/rest"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -80,7 +81,7 @@ var _ = BeforeSuite(func() {
 		Scheme:          k8sManager.GetScheme(),
 		Log:             ctrl.Log.WithName("controllers").WithName("SecretStore"),
 		ControllerClass: defaultControllerClass,
-	}).SetupWithManager(k8sManager)
+	}).SetupWithManager(k8sManager, controller.Options{})
 	Expect(err).ToNot(HaveOccurred())
 
 	err = (&ClusterStoreReconciler{

pkg/provider/aws/auth/auth.go

@@ -382,7 +382,7 @@ func getAWSSession(config *aws.Config, enableCache bool, name, kind, namespace,
 	}
 
 	if enableCache {
-		sessionCache.Add(resourceVersion, key, sess)
+		sessionCache.Add(resourceVersion, key, sess.Copy())
 	}
 	return sess, nil
 }