From ea46ec1911a512ff3f9fbc53250fbd901c31e68c Mon Sep 17 00:00:00 2001 From: Moritz Johner Date: Mon, 12 Jul 2021 20:27:48 +0200 Subject: [PATCH] fix(e2e): refactor e2e tests --- .github/workflows/e2e.yml | 1 + Makefile | 2 +- docs/provider-hashicorp-vault.md | 69 ++++++ e2e/Makefile | 1 + e2e/framework/eso.go | 32 ++- e2e/framework/testcase.go | 91 +++++++ e2e/run.sh | 15 +- e2e/suite/aws/provider.go | 124 ++++++++++ e2e/suite/aws/secretsmanager.go | 218 +---------------- e2e/suite/aws/util.go | 44 ---- e2e/suite/azure/azure.go | 110 +-------- e2e/suite/azure/provider.go | 127 ++++++++++ e2e/suite/azure/util.go | 79 ------ e2e/suite/common/common.go | 230 ++++++++++++++++++ e2e/suite/gcp/gcp.go | 209 +--------------- e2e/suite/gcp/provider.go | 139 +++++++++++ e2e/suite/gcp/util.go | 101 -------- e2e/suite/vault/provider.go | 111 +++++++++ e2e/suite/vault/vault.go | 110 +-------- pkg/provider/azure/keyvault/keyvault.go | 10 +- .../gcp/secretmanager/secretsmanager.go | 4 +- 21 files changed, 980 insertions(+), 847 deletions(-) create mode 100644 e2e/framework/testcase.go create mode 100644 e2e/suite/aws/provider.go delete mode 100644 e2e/suite/aws/util.go create mode 100644 e2e/suite/azure/provider.go delete mode 100644 e2e/suite/azure/util.go create mode 100644 e2e/suite/common/common.go create mode 100644 e2e/suite/gcp/provider.go delete mode 100644 e2e/suite/gcp/util.go create mode 100644 e2e/suite/vault/provider.go diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 5a45a7ffd..6b955885e 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -15,6 +15,7 @@ env: # credentials have been provided before trying to run steps that need them. GHCR_USERNAME: ${{ secrets.GHCR_USERNAME }} GCP_SM_SA_JSON: ${{ secrets.GCP_SM_SA_JSON}} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID}} AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID}} AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET}} TENANT_ID: ${{ secrets.TENANT_ID}} diff --git a/Makefile b/Makefile index 763955575..5d8e2b8ba 100644 --- a/Makefile +++ b/Makefile @@ -78,7 +78,7 @@ check-diff: reviewable .PHONY: test test: generate ## Run tests @$(INFO) go test unit-tests - go test -v $(shell go list ./... | grep -v e2e) -coverprofile cover.out + go test -race -v $(shell go list ./... | grep -v e2e) -coverprofile cover.out @$(OK) go test unit-tests .PHONY: test.e2e diff --git a/docs/provider-hashicorp-vault.md b/docs/provider-hashicorp-vault.md index 99117ac78..6c2f33599 100644 --- a/docs/provider-hashicorp-vault.md +++ b/docs/provider-hashicorp-vault.md @@ -6,6 +6,75 @@ External Secrets Operator integrates with [HashiCorp Vault](https://www.vaultpro management. Vault itself implements lots of different secret engines, as of now we only support the [KV Secrets Engine](https://www.vaultproject.io/docs/secrets/kv). +### Example + +First, create a SecretStore with a vault backend. For the sake of simplicity we'll use a static token `root`: + +```yaml +apiVersion: external-secrets.io/v1alpha1 +kind: SecretStore +metadata: + name: vault-backend +spec: + provider: + vault: + server: "http://my.vault.server:8200" + path: "secret" + version: "v2" + auth: + # points to a secret that contains a vault token + # https://www.vaultproject.io/docs/auth/token + tokenSecretRef: + name: "vault-token" + namespace: "default" + key: "token" +--- +apiVersion: v1 +kind: Secret +metadata: + name: vault-token +data: + token: cm9vdA== # "root" +``` + +Then create a simple k/v pair at path `secret/foo`: + +``` +vault kv put secret/foo my-value=s3cr3t +``` + +Now create a ExternalSecret that uses the above SecretStore: + +```yaml +apiVersion: external-secrets.io/v1alpha1 +kind: ExternalSecret +metadata: + name: vault-example +spec: + refreshInterval: "15s" + secretStoreRef: + name: vault-backend + kind: ClusterSecretStore + target: + name: example-sync + data: + - secretKey: foobar + remoteRef: + key: secret/foo + property: my-value +--- +# will create a secret with: +kind: Secret +metadata: + name: example-sync +data: + foobar: czNjcjN0 +``` + +#### Limitations + +Vault supports only simple key/value pairs - nested objects are not supported. Hence specifying `gjson` properties like other providers support it is not supported. + ### Authentication We support three different modes for authentication: diff --git a/e2e/Makefile b/e2e/Makefile index 1b918c69e..9944403a6 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -5,6 +5,7 @@ SHELL := /bin/bash IMG_TAG = test IMG = local/external-secrets-e2e:$(IMG_TAG) K8S_VERSION = "1.19.1" +export FOCUS := $(FOCUS) start-kind: ## Start kind cluster kind create cluster \ diff --git a/e2e/framework/eso.go b/e2e/framework/eso.go index a1a5ced24..b17d4895c 100644 --- a/e2e/framework/eso.go +++ b/e2e/framework/eso.go @@ -16,6 +16,7 @@ package framework import ( "bytes" "context" + "encoding/json" "time" v1 "k8s.io/api/core/v1" @@ -26,7 +27,7 @@ import ( // WaitForSecretValue waits until a secret comes into existence and compares the secret.Data // with the provided values. -func (f *Framework) WaitForSecretValue(namespace, name string, values map[string][]byte) (*v1.Secret, error) { +func (f *Framework) WaitForSecretValue(namespace, name string, expected *v1.Secret) (*v1.Secret, error) { secret := &v1.Secret{} err := wait.PollImmediate(time.Second*2, time.Minute*2, func() (bool, error) { err := f.CRClient.Get(context.Background(), types.NamespacedName{ @@ -36,13 +37,28 @@ func (f *Framework) WaitForSecretValue(namespace, name string, values map[string if apierrors.IsNotFound(err) { return false, nil } - - for k, exp := range values { - if actual, ok := secret.Data[k]; ok && !bytes.Equal(actual, exp) { - return false, nil - } - } - return true, nil + return equalSecrets(expected, secret), nil }) return secret, err } + +func equalSecrets(exp, ts *v1.Secret) bool { + if exp.Type != ts.Type { + return false + } + expLabels, _ := json.Marshal(exp.ObjectMeta.Labels) + tsLabels, _ := json.Marshal(ts.ObjectMeta.Labels) + if !bytes.Equal(expLabels, tsLabels) { + return false + } + + expAnnotations, _ := json.Marshal(exp.ObjectMeta.Annotations) + tsAnnotations, _ := json.Marshal(ts.ObjectMeta.Annotations) + if !bytes.Equal(expAnnotations, tsAnnotations) { + return false + } + + expData, _ := json.Marshal(exp.Data) + tsData, _ := json.Marshal(ts.Data) + return bytes.Equal(expData, tsData) +} diff --git a/e2e/framework/testcase.go b/e2e/framework/testcase.go new file mode 100644 index 000000000..e4a914af8 --- /dev/null +++ b/e2e/framework/testcase.go @@ -0,0 +1,91 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package framework + +import ( + "context" + + //nolint + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" +) + +const TargetSecretName = "target-secret" + +// TestCase contains the test infra to run a table driven test. +type TestCase struct { + Framework *Framework + ExternalSecret *esv1alpha1.ExternalSecret + Secrets map[string]string + ExpectedSecret *v1.Secret +} + +// SecretStoreProvider is a interface that must be implemented +// by a provider that runs the e2e test. +type SecretStoreProvider interface { + CreateSecret(key string, val string) + DeleteSecret(key string) +} + +// TableFunc returns the main func that runs a TestCase in a table driven test. +func TableFunc(f *Framework, prov SecretStoreProvider) func(func(*TestCase)) { + return func(customize func(*TestCase)) { + var err error + + // make default test case + // and apply customization to it + tc := makeDefaultTestCase(f) + customize(tc) + + // create secrets & defer delete + for k, v := range tc.Secrets { + key := k + prov.CreateSecret(key, v) + defer func() { + prov.DeleteSecret(key) + }() + } + + // create external secret + err = tc.Framework.CRClient.Create(context.Background(), tc.ExternalSecret) + Expect(err).ToNot(HaveOccurred()) + + // wait for Kind=Secret to have the expected data + _, err = tc.Framework.WaitForSecretValue(tc.Framework.Namespace.Name, TargetSecretName, tc.ExpectedSecret) + Expect(err).ToNot(HaveOccurred()) + } +} + +func makeDefaultTestCase(f *Framework) *TestCase { + return &TestCase{ + Framework: f, + ExternalSecret: &esv1alpha1.ExternalSecret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-es", + Namespace: f.Namespace.Name, + }, + Spec: esv1alpha1.ExternalSecretSpec{ + SecretStoreRef: esv1alpha1.SecretStoreRef{ + Name: f.Namespace.Name, + }, + Target: esv1alpha1.ExternalSecretTarget{ + Name: TargetSecretName, + }, + }, + }, + } +} diff --git a/e2e/run.sh b/e2e/run.sh index 54ce7da96..b31121fa7 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -46,18 +46,17 @@ done kubectl apply -f ${DIR}/k8s/deploy/crds echo -e "Starting the e2e test pod" -FOCUS=${FOCUS:-.*} -export FOCUS kubectl run --rm \ --attach \ --restart=Never \ --pod-running-timeout=10m \ - --env="FOCUS=${FOCUS}" \ - --env="GCP_SM_SA_JSON=${GCP_SM_SA_JSON}" \ - --env="AZURE_CLIENT_ID=${AZURE_CLIENT_ID}" \ - --env="AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET}" \ - --env="TENANT_ID=${TENANT_ID}" \ - --env="VAULT_URL=${VAULT_URL}" \ + --env="FOCUS=${FOCUS:-.*}" \ + --env="GCP_SM_SA_JSON=${GCP_SM_SA_JSON:-}" \ + --env="GCP_PROJECT_ID=${GCP_PROJECT_ID:-}" \ + --env="AZURE_CLIENT_ID=${AZURE_CLIENT_ID:-}" \ + --env="AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET:-}" \ + --env="TENANT_ID=${TENANT_ID:-}" \ + --env="VAULT_URL=${VAULT_URL:-}" \ --overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "external-secrets-e2e"}}' \ e2e --image=local/external-secrets-e2e:test diff --git a/e2e/suite/aws/provider.go b/e2e/suite/aws/provider.go new file mode 100644 index 000000000..e603dc9f8 --- /dev/null +++ b/e2e/suite/aws/provider.go @@ -0,0 +1,124 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package aws + +import ( + "context" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/secretsmanager" + + //nolint + . "github.com/onsi/ginkgo" + + // nolint + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" + esmeta "github.com/external-secrets/external-secrets/apis/meta/v1" + "github.com/external-secrets/external-secrets/e2e/framework" + prov "github.com/external-secrets/external-secrets/pkg/provider/aws" +) + +type SMProvider struct { + url string + client *secretsmanager.SecretsManager + framework *framework.Framework +} + +func newSMProvider(f *framework.Framework, url string) *SMProvider { + sess, err := session.NewSessionWithOptions(session.Options{ + Config: aws.Config{ + Credentials: credentials.NewStaticCredentials("foobar", "foobar", "secret-manager"), + EndpointResolver: prov.ResolveEndpointWithServiceMap(map[string]string{ + "secretsmanager": url, + }), + Region: aws.String("eu-east-1"), + }, + }) + Expect(err).ToNot(HaveOccurred()) + sm := secretsmanager.New(sess) + prov := &SMProvider{ + url: url, + client: sm, + framework: f, + } + BeforeEach(prov.BeforeEach) + return prov +} + +func (s *SMProvider) CreateSecret(key, val string) { + _, err := s.client.CreateSecret(&secretsmanager.CreateSecretInput{ + Name: aws.String(key), + SecretString: aws.String(val), + }) + Expect(err).ToNot(HaveOccurred()) +} + +func (s *SMProvider) DeleteSecret(key string) { + _, err := s.client.DeleteSecret(&secretsmanager.DeleteSecretInput{ + SecretId: aws.String(key), + }) + Expect(err).ToNot(HaveOccurred()) +} + +func (s *SMProvider) BeforeEach() { + By("creating a AWS SM credentials secret") + awsCreds := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "provider-secret", + Namespace: s.framework.Namespace.Name, + }, + StringData: map[string]string{ + "kid": "foobar", + "sak": "foobar", + }, + } + err := s.framework.CRClient.Create(context.Background(), awsCreds) + Expect(err).ToNot(HaveOccurred()) + + By("creating a AWS SM secret store") + secretStore := &esv1alpha1.SecretStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: s.framework.Namespace.Name, + Namespace: s.framework.Namespace.Name, + }, + Spec: esv1alpha1.SecretStoreSpec{ + Provider: &esv1alpha1.SecretStoreProvider{ + AWS: &esv1alpha1.AWSProvider{ + Service: esv1alpha1.AWSServiceSecretsManager, + Region: "us-east-1", + Auth: &esv1alpha1.AWSAuth{ + SecretRef: esv1alpha1.AWSAuthSecretRef{ + AccessKeyID: esmeta.SecretKeySelector{ + Name: "provider-secret", + Key: "kid", + }, + SecretAccessKey: esmeta.SecretKeySelector{ + Name: "provider-secret", + Key: "sak", + }, + }, + }, + }, + }, + }, + } + err = s.framework.CRClient.Create(context.Background(), secretStore) + Expect(err).ToNot(HaveOccurred()) +} diff --git a/e2e/suite/aws/secretsmanager.go b/e2e/suite/aws/secretsmanager.go index 20abcb22a..ab43f9fa5 100644 --- a/e2e/suite/aws/secretsmanager.go +++ b/e2e/suite/aws/secretsmanager.go @@ -14,222 +14,26 @@ limitations under the License. package aws import ( - "context" - "fmt" // nolint . "github.com/onsi/ginkgo" // nolint - . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + . "github.com/onsi/ginkgo/extensions/table" - esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" - esmeta "github.com/external-secrets/external-secrets/apis/meta/v1" "github.com/external-secrets/external-secrets/e2e/framework" -) - -const ( - targetSecret = "target-secret" + "github.com/external-secrets/external-secrets/e2e/suite/common" ) var _ = Describe("[aws] ", func() { f := framework.New("eso-aws") - var secretStore *esv1alpha1.SecretStore - localstackURL := "http://localstack.default" - BeforeEach(func() { - By("creating an secret store for localstack") - awsCreds := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: f.Namespace.Name, - Namespace: f.Namespace.Name, - }, - StringData: map[string]string{ - "kid": "foobar", - "sak": "foobar", - }, - } - err := f.CRClient.Create(context.Background(), awsCreds) - Expect(err).ToNot(HaveOccurred()) - secretStore = &esv1alpha1.SecretStore{ - ObjectMeta: metav1.ObjectMeta{ - Name: f.Namespace.Name, - Namespace: f.Namespace.Name, - }, - Spec: esv1alpha1.SecretStoreSpec{ - Provider: &esv1alpha1.SecretStoreProvider{ - AWS: &esv1alpha1.AWSProvider{ - Service: esv1alpha1.AWSServiceSecretsManager, - Region: "us-east-1", - Auth: &esv1alpha1.AWSAuth{ - SecretRef: esv1alpha1.AWSAuthSecretRef{ - AccessKeyID: esmeta.SecretKeySelector{ - Name: f.Namespace.Name, - Key: "kid", - }, - SecretAccessKey: esmeta.SecretKeySelector{ - Name: f.Namespace.Name, - Key: "sak", - }, - }, - }, - }, - }, - }, - } - err = f.CRClient.Create(context.Background(), secretStore) - Expect(err).ToNot(HaveOccurred()) - }) - - It("should sync multiple secrets", func() { - By("creating a AWS SM Secret") - secretKey1 := fmt.Sprintf("%s-%s", f.Namespace.Name, "one") - secretKey2 := fmt.Sprintf("%s-%s", f.Namespace.Name, "other") - secretValue := "bar" - err := CreateAWSSecretsManagerSecret( - localstackURL, - secretKey1, secretValue) - Expect(err).ToNot(HaveOccurred()) - err = CreateAWSSecretsManagerSecret( - localstackURL, - secretKey2, secretValue) - Expect(err).ToNot(HaveOccurred()) - - err = f.CRClient.Create(context.Background(), &esv1alpha1.ExternalSecret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "simple-sync", - Namespace: f.Namespace.Name, - }, - Spec: esv1alpha1.ExternalSecretSpec{ - SecretStoreRef: esv1alpha1.SecretStoreRef{ - Name: f.Namespace.Name, - }, - Target: esv1alpha1.ExternalSecretTarget{ - Name: targetSecret, - }, - Data: []esv1alpha1.ExternalSecretData{ - { - SecretKey: secretKey1, - RemoteRef: esv1alpha1.ExternalSecretDataRemoteRef{ - Key: secretKey1, - }, - }, - { - SecretKey: secretKey2, - RemoteRef: esv1alpha1.ExternalSecretDataRemoteRef{ - Key: secretKey2, - }, - }, - }, - }, - }) - Expect(err).ToNot(HaveOccurred()) - - _, err = f.WaitForSecretValue(f.Namespace.Name, targetSecret, map[string][]byte{ - secretKey1: []byte(secretValue), - secretKey2: []byte(secretValue), - }) - Expect(err).ToNot(HaveOccurred()) - }) - - It("should sync secrets with dataFrom", func() { - By("creating a AWS SM Secret") - secretKey1 := fmt.Sprintf("%s-%s", f.Namespace.Name, "one") - targetSecretKey1 := "name" - targetSecretValue1 := "great-name" - targetSecretKey2 := "surname" - targetSecretValue2 := "great-surname" - secretValue := fmt.Sprintf("{ \"%s\": \"%s\", \"%s\": \"%s\" }", targetSecretKey1, targetSecretValue1, targetSecretKey2, targetSecretValue2) - err := CreateAWSSecretsManagerSecret( - localstackURL, - secretKey1, secretValue) - Expect(err).ToNot(HaveOccurred()) - err = f.CRClient.Create(context.Background(), &esv1alpha1.ExternalSecret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "datafrom-sync", - Namespace: f.Namespace.Name, - }, - Spec: esv1alpha1.ExternalSecretSpec{ - SecretStoreRef: esv1alpha1.SecretStoreRef{ - Name: f.Namespace.Name, - }, - Target: esv1alpha1.ExternalSecretTarget{ - Name: targetSecret, - }, - DataFrom: []esv1alpha1.ExternalSecretDataRemoteRef{ - { - Key: secretKey1, - }, - }, - }, - }) - Expect(err).ToNot(HaveOccurred()) - - _, err = f.WaitForSecretValue(f.Namespace.Name, targetSecret, map[string][]byte{ - targetSecretKey1: []byte(targetSecretValue1), - targetSecretKey2: []byte(targetSecretValue2), - }) - Expect(err).ToNot(HaveOccurred()) - }) - - It("should sync secrets and get inner keys", func() { - By("creating a AWS SM Secret") - secretKey1 := fmt.Sprintf("%s-%s", f.Namespace.Name, "one") - targetSecretKey1 := "firstname" - targetSecretValue1 := "Tom" - targetSecretKey2 := "first_friend" - targetSecretValue2 := "Roger" - secretValue := fmt.Sprintf( - `{ - "name": {"first": "%s", "last": "Anderson"}, - "friends": - [ - {"first": "Dale", "last": "Murphy"}, - {"first": "%s", "last": "Craig"}, - {"first": "Jane", "last": "Murphy"} - ] - }`, targetSecretValue1, targetSecretValue2) - err := CreateAWSSecretsManagerSecret( - localstackURL, - secretKey1, secretValue) - Expect(err).ToNot(HaveOccurred()) - err = f.CRClient.Create(context.Background(), &esv1alpha1.ExternalSecret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "datafrom-sync", - Namespace: f.Namespace.Name, - }, - Spec: esv1alpha1.ExternalSecretSpec{ - SecretStoreRef: esv1alpha1.SecretStoreRef{ - Name: f.Namespace.Name, - }, - Target: esv1alpha1.ExternalSecretTarget{ - Name: targetSecret, - }, - Data: []esv1alpha1.ExternalSecretData{ - { - SecretKey: targetSecretKey1, - RemoteRef: esv1alpha1.ExternalSecretDataRemoteRef{ - Key: secretKey1, - Property: "name.first", - }, - }, - { - SecretKey: targetSecretKey2, - RemoteRef: esv1alpha1.ExternalSecretDataRemoteRef{ - Key: secretKey1, - Property: "friends.1.first", - }, - }, - }, - }, - }) - Expect(err).ToNot(HaveOccurred()) - - _, err = f.WaitForSecretValue(f.Namespace.Name, targetSecret, map[string][]byte{ - targetSecretKey1: []byte(targetSecretValue1), - targetSecretKey2: []byte(targetSecretValue2), - }) - Expect(err).ToNot(HaveOccurred()) - }) + DescribeTable("sync secrets", + framework.TableFunc(f, + newSMProvider(f, "http://localstack.default")), + Entry(common.SimpleDataSync(f)), + Entry(common.NestedJSONWithGJSON(f)), + Entry(common.JSONDataFromSync(f)), + Entry(common.JSONDataWithProperty(f)), + Entry(common.JSONDataWithTemplate(f)), + ) }) diff --git a/e2e/suite/aws/util.go b/e2e/suite/aws/util.go deleted file mode 100644 index 741ca71e4..000000000 --- a/e2e/suite/aws/util.go +++ /dev/null @@ -1,44 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -limitations under the License. -*/ -package aws - -import ( - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/secretsmanager" - - prov "github.com/external-secrets/external-secrets/pkg/provider/aws" -) - -// CreateAWSSecretsManagerSecret creates a sm secret with the given value. -func CreateAWSSecretsManagerSecret(endpoint, secretName, secretValue string) error { - sess, err := session.NewSessionWithOptions(session.Options{ - Config: aws.Config{ - Credentials: credentials.NewStaticCredentials("foobar", "foobar", "secret-manager"), - EndpointResolver: prov.ResolveEndpointWithServiceMap(map[string]string{ - "secretsmanager": endpoint, - }), - Region: aws.String("eu-east-1"), - }, - }) - if err != nil { - return err - } - sm := secretsmanager.New(sess) - _, err = sm.CreateSecret(&secretsmanager.CreateSecretInput{ - Name: aws.String(secretName), - SecretString: aws.String(secretValue), - }) - return err -} diff --git a/e2e/suite/azure/azure.go b/e2e/suite/azure/azure.go index 529b908bb..a2ffd1c2f 100644 --- a/e2e/suite/azure/azure.go +++ b/e2e/suite/azure/azure.go @@ -13,118 +13,32 @@ limitations under the License. package azure import ( - "context" - "fmt" "os" // nolint . "github.com/onsi/ginkgo" // nolint - . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + . "github.com/onsi/ginkgo/extensions/table" - esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" - esmeta "github.com/external-secrets/external-secrets/apis/meta/v1" "github.com/external-secrets/external-secrets/e2e/framework" -) - -const ( - targetSecret = "target-secret" + "github.com/external-secrets/external-secrets/e2e/suite/common" ) var _ = Describe("[azure] ", func() { f := framework.New("eso-azure") - var secretStore *esv1alpha1.SecretStore vaultURL := os.Getenv("VAULT_URL") tenantID := os.Getenv("TENANT_ID") clientID := os.Getenv("AZURE_CLIENT_ID") clientSecret := os.Getenv("AZURE_CLIENT_SECRET") - BeforeEach(func() { - By("creating a secret in AzureKV") - azureCreds := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: f.Namespace.Name, - Namespace: f.Namespace.Name, - }, - StringData: map[string]string{ - "ClientID": clientID, - "ClientSecret": clientSecret, - }, - } - err := f.CRClient.Create(context.Background(), azureCreds) - Expect(err).ToNot(HaveOccurred()) - secretStore = &esv1alpha1.SecretStore{ - ObjectMeta: metav1.ObjectMeta{ - Name: f.Namespace.Name, - Namespace: f.Namespace.Name, - }, - Spec: esv1alpha1.SecretStoreSpec{ - Provider: &esv1alpha1.SecretStoreProvider{ - AzureKV: &esv1alpha1.AzureKVProvider{ - TenantID: &tenantID, - VaultURL: &vaultURL, - AuthSecretRef: &esv1alpha1.AzureKVAuth{ - ClientID: &esmeta.SecretKeySelector{ - Name: f.Namespace.Name, - Key: "ClientID", - }, - ClientSecret: &esmeta.SecretKeySelector{ - Name: f.Namespace.Name, - Key: "ClientSecret", - }, - }, - }, - }, - }, - } - err = f.CRClient.Create(context.Background(), secretStore) - Expect(err).ToNot(HaveOccurred()) - }) - - It("should sync secrets", func() { - By("creating a AzureKV Secret") - secretKey1 := fmt.Sprintf("%s-%s", f.Namespace.Name, "one") - secretValue := "great-value-test" - _, err := createAzureKVSecret( - secretKey1, - secretValue, - clientID, - clientSecret, - tenantID, - vaultURL) - Expect(err).ToNot(HaveOccurred()) - err = f.CRClient.Create(context.Background(), &esv1alpha1.ExternalSecret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "simple-sync", - Namespace: f.Namespace.Name, - }, - Spec: esv1alpha1.ExternalSecretSpec{ - SecretStoreRef: esv1alpha1.SecretStoreRef{ - Name: f.Namespace.Name, - }, - Target: esv1alpha1.ExternalSecretTarget{ - Name: targetSecret, - }, - Data: []esv1alpha1.ExternalSecretData{ - { - SecretKey: secretKey1, - RemoteRef: esv1alpha1.ExternalSecretDataRemoteRef{ - Key: secretKey1, - }, - }, - }, - }, - }) - Expect(err).ToNot(HaveOccurred()) - - _, err = f.WaitForSecretValue(f.Namespace.Name, targetSecret, map[string][]byte{ - secretKey1: []byte(secretValue), - }) - Expect(err).ToNot(HaveOccurred()) - - err = deleteAzureKVSecret(secretKey1, clientID, clientSecret, tenantID, vaultURL) - Expect(err).ToNot(HaveOccurred()) - }) + prov := newazureProvider(f, clientID, clientSecret, tenantID, vaultURL) + DescribeTable("sync secrets", framework.TableFunc(f, prov), + Entry(common.SimpleDataSync(f)), + Entry(common.NestedJSONWithGJSON(f)), + // TODO: dataFrom is not working as expected RN + // see: https://github.com/external-secrets/external-secrets/issues/263 + // Entry(common.JSONDataFromSync(f)), + Entry(common.JSONDataWithProperty(f)), + Entry(common.JSONDataWithTemplate(f)), + ) }) diff --git a/e2e/suite/azure/provider.go b/e2e/suite/azure/provider.go new file mode 100644 index 000000000..a3d312303 --- /dev/null +++ b/e2e/suite/azure/provider.go @@ -0,0 +1,127 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +limitations under the License. +*/ +package azure + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/profiles/latest/keyvault/keyvault" + kvauth "github.com/Azure/go-autorest/autorest/azure/auth" + + // nolint + . "github.com/onsi/ginkgo" + + // nolint + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilpointer "k8s.io/utils/pointer" + + esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" + esmeta "github.com/external-secrets/external-secrets/apis/meta/v1" + "github.com/external-secrets/external-secrets/e2e/framework" +) + +type azureProvider struct { + clientID string + clientSecret string + tenantID string + vaultURL string + client *keyvault.BaseClient + framework *framework.Framework +} + +func newazureProvider(f *framework.Framework, clientID, clientSecret, tenantID, vaultURL string) *azureProvider { + clientCredentialsConfig := kvauth.NewClientCredentialsConfig(clientID, clientSecret, tenantID) + clientCredentialsConfig.Resource = "https://vault.azure.net" + authorizer, err := clientCredentialsConfig.Authorizer() + Expect(err).ToNot(HaveOccurred()) + basicClient := keyvault.New() + basicClient.Authorizer = authorizer + + prov := &azureProvider{ + framework: f, + clientID: clientID, + clientSecret: clientSecret, + tenantID: tenantID, + vaultURL: vaultURL, + client: &basicClient, + } + BeforeEach(prov.BeforeEach) + return prov +} + +func (s *azureProvider) CreateSecret(key, val string) { + _, err := s.client.SetSecret( + context.Background(), + s.vaultURL, + key, + keyvault.SecretSetParameters{ + Value: &val, + SecretAttributes: &keyvault.SecretAttributes{ + RecoveryLevel: keyvault.Purgeable, + Enabled: utilpointer.BoolPtr(true), + }, + }) + Expect(err).ToNot(HaveOccurred()) +} + +func (s *azureProvider) DeleteSecret(key string) { + _, err := s.client.DeleteSecret( + context.Background(), + s.vaultURL, + key) + Expect(err).ToNot(HaveOccurred()) +} + +func (s *azureProvider) BeforeEach() { + azureCreds := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "provider-secret", + Namespace: s.framework.Namespace.Name, + }, + StringData: map[string]string{ + "client-id": s.clientID, + "client-secret": s.clientSecret, + }, + } + err := s.framework.CRClient.Create(context.Background(), azureCreds) + Expect(err).ToNot(HaveOccurred()) + + secretStore := &esv1alpha1.SecretStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: s.framework.Namespace.Name, + Namespace: s.framework.Namespace.Name, + }, + Spec: esv1alpha1.SecretStoreSpec{ + Provider: &esv1alpha1.SecretStoreProvider{ + AzureKV: &esv1alpha1.AzureKVProvider{ + TenantID: &s.tenantID, + VaultURL: &s.vaultURL, + AuthSecretRef: &esv1alpha1.AzureKVAuth{ + ClientID: &esmeta.SecretKeySelector{ + Name: "provider-secret", + Key: "client-id", + }, + ClientSecret: &esmeta.SecretKeySelector{ + Name: "provider-secret", + Key: "client-secret", + }, + }, + }, + }, + }, + } + err = s.framework.CRClient.Create(context.Background(), secretStore) + Expect(err).ToNot(HaveOccurred()) +} diff --git a/e2e/suite/azure/util.go b/e2e/suite/azure/util.go deleted file mode 100644 index ff37f539c..000000000 --- a/e2e/suite/azure/util.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -limitations under the License. -*/ -package azure - -import ( - "context" - "fmt" - - "github.com/Azure/azure-sdk-for-go/profiles/latest/keyvault/keyvault" - kvauth "github.com/Azure/go-autorest/autorest/azure/auth" - utilpointer "k8s.io/utils/pointer" -) - -// CreateAWSSecretsManagerSecret creates a sm secret with the given value. -func createAzureKVSecret(secretName, secretValue, clientID, clientSecret, tenantID, vaultURL string) (result keyvault.SecretBundle, err error) { - ctx := context.Background() - - clientCredentialsConfig := kvauth.NewClientCredentialsConfig(clientID, clientSecret, tenantID) - clientCredentialsConfig.Resource = "https://vault.azure.net" - authorizer, err := clientCredentialsConfig.Authorizer() - if err != nil { - return keyvault.SecretBundle{}, fmt.Errorf("could not configure azure authorizer: %w", err) - } - - basicClient := keyvault.New() - basicClient.Authorizer = authorizer - deletionRecoveryLevel := keyvault.Purgeable - result, err = basicClient.SetSecret( - ctx, - vaultURL, - secretName, - keyvault.SecretSetParameters{ - Value: &secretValue, - SecretAttributes: &keyvault.SecretAttributes{ - RecoveryLevel: deletionRecoveryLevel, - Enabled: utilpointer.BoolPtr(true), - }, - }) - if err != nil { - return keyvault.SecretBundle{}, fmt.Errorf("could not create secret key %s: %w", secretName, err) - } - - return result, err -} - -// deleteSecret deletes the secret with the given name and all of its versions. -func deleteAzureKVSecret(secretName, clientID, clientSecret, tenantID, vaultURL string) error { - ctx := context.Background() - - clientCredentialsConfig := kvauth.NewClientCredentialsConfig(clientID, clientSecret, tenantID) - clientCredentialsConfig.Resource = "https://vault.azure.net" - authorizer, err := clientCredentialsConfig.Authorizer() - if err != nil { - return fmt.Errorf("could not configure azure authorizer: %w", err) - } - - basicClient := keyvault.New() - basicClient.Authorizer = authorizer - - _, err = basicClient.DeleteSecret( - ctx, - vaultURL, - secretName) - if err != nil { - return fmt.Errorf("could not delete secret: %w", err) - } - - return err -} diff --git a/e2e/suite/common/common.go b/e2e/suite/common/common.go new file mode 100644 index 000000000..85d785412 --- /dev/null +++ b/e2e/suite/common/common.go @@ -0,0 +1,230 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +limitations under the License. +*/ +package common + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" + "github.com/external-secrets/external-secrets/e2e/framework" +) + +// This case creates multiple secrets with simple key/value pairs and syncs them using multiple .Spec.Data blocks. +// Not supported by: vault. +func SimpleDataSync(f *framework.Framework) (string, func(*framework.TestCase)) { + return "[common] should sync simple secrets from .Data[]", func(tc *framework.TestCase) { + secretKey1 := fmt.Sprintf("%s-%s", f.Namespace.Name, "one") + secretKey2 := fmt.Sprintf("%s-%s", f.Namespace.Name, "other") + secretValue := "bar" + tc.Secrets = map[string]string{ + secretKey1: secretValue, + secretKey2: secretValue, + } + tc.ExpectedSecret = &v1.Secret{ + Type: v1.SecretTypeOpaque, + Data: map[string][]byte{ + secretKey1: []byte(secretValue), + secretKey2: []byte(secretValue), + }, + } + tc.ExternalSecret.Spec.Data = []esv1alpha1.ExternalSecretData{ + { + SecretKey: secretKey1, + RemoteRef: esv1alpha1.ExternalSecretDataRemoteRef{ + Key: secretKey1, + }, + }, + { + SecretKey: secretKey2, + RemoteRef: esv1alpha1.ExternalSecretDataRemoteRef{ + Key: secretKey2, + }, + }, + } + } +} + +// This case creates multiple secrets with json values and syncs them using multiple .Spec.Data blocks. +// The data is extracted from the JSON key using ref.Property. +func JSONDataWithProperty(f *framework.Framework) (string, func(*framework.TestCase)) { + return "[common] should sync multiple secrets from .Data[]", func(tc *framework.TestCase) { + secretKey1 := fmt.Sprintf("%s-%s", f.Namespace.Name, "one") + secretKey2 := fmt.Sprintf("%s-%s", f.Namespace.Name, "two") + secretValue1 := "{\"foo1\":\"foo1-val\",\"bar1\":\"bar1-val\"}" + secretValue2 := "{\"foo2\":\"foo2-val\",\"bar2\":\"bar2-val\"}" + tc.Secrets = map[string]string{ + secretKey1: secretValue1, + secretKey2: secretValue2, + } + tc.ExpectedSecret = &v1.Secret{ + Type: v1.SecretTypeOpaque, + Data: map[string][]byte{ + secretKey1: []byte("foo1-val"), + secretKey2: []byte("bar2-val"), + }, + } + tc.ExternalSecret.Spec.Data = []esv1alpha1.ExternalSecretData{ + { + SecretKey: secretKey1, + RemoteRef: esv1alpha1.ExternalSecretDataRemoteRef{ + Key: secretKey1, + Property: "foo1", + }, + }, + { + SecretKey: secretKey2, + RemoteRef: esv1alpha1.ExternalSecretDataRemoteRef{ + Key: secretKey2, + Property: "bar2", + }, + }, + } + } +} + +// This case creates multiple secrets with json values and renders a template. +// The data is extracted from the JSON key using ref.Property. +func JSONDataWithTemplate(f *framework.Framework) (string, func(*framework.TestCase)) { + return "[common] should sync json secrets with template", func(tc *framework.TestCase) { + secretKey1 := fmt.Sprintf("%s-%s", f.Namespace.Name, "one") + secretKey2 := fmt.Sprintf("%s-%s", f.Namespace.Name, "other") + secretValue1 := "{\"foo1\":\"foo1-val\",\"bar1\":\"bar1-val\"}" + secretValue2 := "{\"foo2\":\"foo2-val\",\"bar2\":\"bar2-val\"}" + tc.Secrets = map[string]string{ + secretKey1: secretValue1, + secretKey2: secretValue2, + } + tc.ExpectedSecret = &v1.Secret{ + Type: v1.SecretTypeOpaque, + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "example": "annotation", + }, + Labels: map[string]string{ + "example": "label", + }, + }, + Data: map[string][]byte{ + "my-data": []byte(`executed: foo1-val|bar2-val`), + }, + } + tc.ExternalSecret.Spec.Target.Template = &esv1alpha1.ExternalSecretTemplate{ + Metadata: esv1alpha1.ExternalSecretTemplateMetadata{ + Annotations: map[string]string{ + "example": "annotation", + }, + Labels: map[string]string{ + "example": "label", + }, + }, + Data: map[string]string{ + "my-data": "executed: {{ .one | toString }}|{{ .two | toString }}", + }, + } + tc.ExternalSecret.Spec.Data = []esv1alpha1.ExternalSecretData{ + { + SecretKey: "one", + RemoteRef: esv1alpha1.ExternalSecretDataRemoteRef{ + Key: secretKey1, + Property: "foo1", + }, + }, + { + SecretKey: "two", + RemoteRef: esv1alpha1.ExternalSecretDataRemoteRef{ + Key: secretKey2, + Property: "bar2", + }, + }, + } + } +} + +// This case creates one secret with json values and syncs them using a single .Spec.DataFrom block. +func JSONDataFromSync(f *framework.Framework) (string, func(*framework.TestCase)) { + return "[common] should sync secrets with dataFrom", func(tc *framework.TestCase) { + secretKey1 := fmt.Sprintf("%s-%s", f.Namespace.Name, "one") + targetSecretKey1 := "name" + targetSecretValue1 := "great-name" + targetSecretKey2 := "surname" + targetSecretValue2 := "great-surname" + secretValue := fmt.Sprintf("{ \"%s\": \"%s\", \"%s\": \"%s\" }", targetSecretKey1, targetSecretValue1, targetSecretKey2, targetSecretValue2) + tc.Secrets = map[string]string{ + secretKey1: secretValue, + } + tc.ExpectedSecret = &v1.Secret{ + Type: v1.SecretTypeOpaque, + Data: map[string][]byte{ + targetSecretKey1: []byte(targetSecretValue1), + targetSecretKey2: []byte(targetSecretValue2), + }, + } + tc.ExternalSecret.Spec.DataFrom = []esv1alpha1.ExternalSecretDataRemoteRef{ + { + Key: secretKey1, + }, + } + } +} + +// This case creates a secret with a nested json value. It is synced into two secrets. +// The values from the nested data are extracted using gjson. +// not supported by: vault. +func NestedJSONWithGJSON(f *framework.Framework) (string, func(*framework.TestCase)) { + return "[common] should sync nested json secrets and get inner keys", func(tc *framework.TestCase) { + secretKey1 := fmt.Sprintf("%s-%s", f.Namespace.Name, "one") + targetSecretKey1 := "firstname" + targetSecretValue1 := "Tom" + targetSecretKey2 := "first_friend" + targetSecretValue2 := "Roger" + secretValue := fmt.Sprintf( + `{ + "name": {"first": "%s", "last": "Anderson"}, + "friends": + [ + {"first": "Dale", "last": "Murphy"}, + {"first": "%s", "last": "Craig"}, + {"first": "Jane", "last": "Murphy"} + ] + }`, targetSecretValue1, targetSecretValue2) + tc.Secrets = map[string]string{ + secretKey1: secretValue, + } + tc.ExpectedSecret = &v1.Secret{ + Type: v1.SecretTypeOpaque, + Data: map[string][]byte{ + targetSecretKey1: []byte(targetSecretValue1), + targetSecretKey2: []byte(targetSecretValue2), + }, + } + tc.ExternalSecret.Spec.Data = []esv1alpha1.ExternalSecretData{ + { + SecretKey: targetSecretKey1, + RemoteRef: esv1alpha1.ExternalSecretDataRemoteRef{ + Key: secretKey1, + Property: "name.first", + }, + }, + { + SecretKey: targetSecretKey2, + RemoteRef: esv1alpha1.ExternalSecretDataRemoteRef{ + Key: secretKey1, + Property: "friends.1.first", + }, + }, + } + } +} diff --git a/e2e/suite/gcp/gcp.go b/e2e/suite/gcp/gcp.go index 507b8b204..4b69f61fc 100644 --- a/e2e/suite/gcp/gcp.go +++ b/e2e/suite/gcp/gcp.go @@ -13,215 +13,28 @@ limitations under the License. package gcp import ( - "context" - "fmt" "os" // nolint . "github.com/onsi/ginkgo" // nolint - . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + . "github.com/onsi/ginkgo/extensions/table" - esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" - esmeta "github.com/external-secrets/external-secrets/apis/meta/v1" "github.com/external-secrets/external-secrets/e2e/framework" -) - -const ( - targetSecret = "target-secret" + "github.com/external-secrets/external-secrets/e2e/suite/common" ) var _ = Describe("[gcp] ", func() { f := framework.New("eso-gcp") - var secretStore *esv1alpha1.SecretStore - projectID := "external-secrets-operator" credentials := os.Getenv("GCP_SM_SA_JSON") + projectID := os.Getenv("GCP_PROJECT_ID") + prov := newgcpProvider(f, credentials, projectID) - BeforeEach(func() { - By("creating a secret in GCP SM") - gcpCred := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: f.Namespace.Name, - Namespace: f.Namespace.Name, - }, - StringData: map[string]string{ - "secret-access-credentials": credentials, - }, - } - err := f.CRClient.Create(context.Background(), gcpCred) - Expect(err).ToNot(HaveOccurred()) - secretStore = &esv1alpha1.SecretStore{ - ObjectMeta: metav1.ObjectMeta{ - Name: f.Namespace.Name, - Namespace: f.Namespace.Name, - }, - Spec: esv1alpha1.SecretStoreSpec{ - Provider: &esv1alpha1.SecretStoreProvider{ - GCPSM: &esv1alpha1.GCPSMProvider{ - ProjectID: projectID, - Auth: esv1alpha1.GCPSMAuth{ - SecretRef: esv1alpha1.GCPSMAuthSecretRef{ - SecretAccessKey: esmeta.SecretKeySelector{ - Name: f.Namespace.Name, - Key: "secret-access-credentials", - }, - }, - }, - }, - }, - }, - } - err = f.CRClient.Create(context.Background(), secretStore) - Expect(err).ToNot(HaveOccurred()) - }) - - It("should sync secrets", func() { - By("creating a GCP SM Secret") - secretKey1 := fmt.Sprintf("%s-%s", f.Namespace.Name, "one") - secretValue := "great-value-test" - secret, err := createGCPSecretsManagerSecret( - projectID, - secretKey1, secretValue, []byte(credentials)) - Expect(err).ToNot(HaveOccurred()) - err = f.CRClient.Create(context.Background(), &esv1alpha1.ExternalSecret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "simple-sync", - Namespace: f.Namespace.Name, - }, - Spec: esv1alpha1.ExternalSecretSpec{ - SecretStoreRef: esv1alpha1.SecretStoreRef{ - Name: f.Namespace.Name, - }, - Target: esv1alpha1.ExternalSecretTarget{ - Name: targetSecret, - }, - Data: []esv1alpha1.ExternalSecretData{ - { - SecretKey: secretKey1, - RemoteRef: esv1alpha1.ExternalSecretDataRemoteRef{ - Key: secretKey1, - }, - }, - }, - }, - }) - Expect(err).ToNot(HaveOccurred()) - - _, err = f.WaitForSecretValue(f.Namespace.Name, targetSecret, map[string][]byte{ - secretKey1: []byte(secretValue), - }) - Expect(err).ToNot(HaveOccurred()) - - err = deleteGCPSecretsManagerSecret(secret.Name, []byte(credentials)) - Expect(err).ToNot(HaveOccurred()) - }) - - It("should sync secrets with dataFrom", func() { - By("creating a GCP SM Secret with JSON string") - secretKey1 := fmt.Sprintf("%s-%s", f.Namespace.Name, "one") - targetSecretKey1 := "name" - targetSecretValue1 := "great-name" - targetSecretKey2 := "surname" - targetSecretValue2 := "great-surname" - secretValue := fmt.Sprintf("{ \"%s\": \"%s\", \"%s\": \"%s\" }", targetSecretKey1, targetSecretValue1, targetSecretKey2, targetSecretValue2) - secret, err := createGCPSecretsManagerSecret( - projectID, - secretKey1, secretValue, []byte(credentials)) - Expect(err).ToNot(HaveOccurred()) - err = f.CRClient.Create(context.Background(), &esv1alpha1.ExternalSecret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "datafrom-sync", - Namespace: f.Namespace.Name, - }, - Spec: esv1alpha1.ExternalSecretSpec{ - SecretStoreRef: esv1alpha1.SecretStoreRef{ - Name: f.Namespace.Name, - }, - Target: esv1alpha1.ExternalSecretTarget{ - Name: targetSecret, - }, - DataFrom: []esv1alpha1.ExternalSecretDataRemoteRef{ - { - Key: secretKey1, - }, - }, - }, - }) - Expect(err).ToNot(HaveOccurred()) - - _, err = f.WaitForSecretValue(f.Namespace.Name, targetSecret, map[string][]byte{ - targetSecretKey1: []byte(targetSecretValue1), - targetSecretKey2: []byte(targetSecretValue2), - }) - Expect(err).ToNot(HaveOccurred()) - - err = deleteGCPSecretsManagerSecret(secret.Name, []byte(credentials)) - Expect(err).ToNot(HaveOccurred()) - }) - - It("should sync secrets and get inner keys", func() { - By("creating a GCP SM Secret") - secretKey1 := fmt.Sprintf("%s-%s", f.Namespace.Name, "one") - targetSecretKey1 := "firstname" - targetSecretValue1 := "Tom" - targetSecretKey2 := "first_friend" - targetSecretValue2 := "Roger" - secretValue := fmt.Sprintf( - `{ - "name": {"first": "%s", "last": "Anderson"}, - "friends": - [ - {"first": "Dale", "last": "Murphy"}, - {"first": "%s", "last": "Craig"}, - {"first": "Jane", "last": "Murphy"} - ] - }`, targetSecretValue1, targetSecretValue2) - secret, err := createGCPSecretsManagerSecret( - projectID, - secretKey1, secretValue, []byte(credentials)) - Expect(err).ToNot(HaveOccurred()) - err = f.CRClient.Create(context.Background(), &esv1alpha1.ExternalSecret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "datafrom-sync", - Namespace: f.Namespace.Name, - }, - Spec: esv1alpha1.ExternalSecretSpec{ - SecretStoreRef: esv1alpha1.SecretStoreRef{ - Name: f.Namespace.Name, - }, - Target: esv1alpha1.ExternalSecretTarget{ - Name: targetSecret, - }, - Data: []esv1alpha1.ExternalSecretData{ - { - SecretKey: targetSecretKey1, - RemoteRef: esv1alpha1.ExternalSecretDataRemoteRef{ - Key: secretKey1, - Property: "name.first", - }, - }, - { - SecretKey: targetSecretKey2, - RemoteRef: esv1alpha1.ExternalSecretDataRemoteRef{ - Key: secretKey1, - Property: "friends.1.first", - }, - }, - }, - }, - }) - Expect(err).ToNot(HaveOccurred()) - - _, err = f.WaitForSecretValue(f.Namespace.Name, targetSecret, map[string][]byte{ - targetSecretKey1: []byte(targetSecretValue1), - targetSecretKey2: []byte(targetSecretValue2), - }) - Expect(err).ToNot(HaveOccurred()) - - err = deleteGCPSecretsManagerSecret(secret.Name, []byte(credentials)) - Expect(err).ToNot(HaveOccurred()) - }) - + DescribeTable("sync secrets", framework.TableFunc(f, prov), + Entry(common.SimpleDataSync(f)), + Entry(common.JSONDataWithProperty(f)), + Entry(common.JSONDataFromSync(f)), + Entry(common.NestedJSONWithGJSON(f)), + Entry(common.JSONDataWithTemplate(f)), + ) }) diff --git a/e2e/suite/gcp/provider.go b/e2e/suite/gcp/provider.go new file mode 100644 index 000000000..48c310610 --- /dev/null +++ b/e2e/suite/gcp/provider.go @@ -0,0 +1,139 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +limitations under the License. +*/ +package gcp + +import ( + "context" + "fmt" + + secretmanager "cloud.google.com/go/secretmanager/apiv1" + + // nolint + . "github.com/onsi/ginkgo" + + // nolint + . "github.com/onsi/gomega" + "golang.org/x/oauth2/google" + "google.golang.org/api/option" + secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" + esmeta "github.com/external-secrets/external-secrets/apis/meta/v1" + "github.com/external-secrets/external-secrets/e2e/framework" + gcpsm "github.com/external-secrets/external-secrets/pkg/provider/gcp/secretmanager" +) + +type gcpProvider struct { + credentials string + projectID string + framework *framework.Framework +} + +func newgcpProvider(f *framework.Framework, credentials, projectID string) *gcpProvider { + prov := &gcpProvider{ + credentials: credentials, + projectID: projectID, + framework: f, + } + BeforeEach(prov.BeforeEach) + return prov +} + +func (s *gcpProvider) CreateSecret(key, val string) { + ctx := context.Background() + config, err := google.JWTConfigFromJSON([]byte(s.credentials), gcpsm.CloudPlatformRole) + Expect(err).ToNot(HaveOccurred()) + ts := config.TokenSource(ctx) + client, err := secretmanager.NewClient(ctx, option.WithTokenSource(ts)) + Expect(err).ToNot(HaveOccurred()) + defer client.Close() + // Create the request to create the secret. + createSecretReq := &secretmanagerpb.CreateSecretRequest{ + Parent: fmt.Sprintf("projects/%s", s.projectID), + SecretId: key, + Secret: &secretmanagerpb.Secret{ + Replication: &secretmanagerpb.Replication{ + Replication: &secretmanagerpb.Replication_Automatic_{ + Automatic: &secretmanagerpb.Replication_Automatic{}, + }, + }, + }, + } + secret, err := client.CreateSecret(ctx, createSecretReq) + Expect(err).ToNot(HaveOccurred()) + addSecretVersionReq := &secretmanagerpb.AddSecretVersionRequest{ + Parent: secret.Name, + Payload: &secretmanagerpb.SecretPayload{ + Data: []byte(val), + }, + } + _, err = client.AddSecretVersion(ctx, addSecretVersionReq) + Expect(err).ToNot(HaveOccurred()) +} + +func (s *gcpProvider) DeleteSecret(key string) { + ctx := context.Background() + config, err := google.JWTConfigFromJSON([]byte(s.credentials), gcpsm.CloudPlatformRole) + Expect(err).ToNot(HaveOccurred()) + ts := config.TokenSource(ctx) + client, err := secretmanager.NewClient(ctx, option.WithTokenSource(ts)) + Expect(err).ToNot(HaveOccurred()) + defer client.Close() + req := &secretmanagerpb.DeleteSecretRequest{ + Name: fmt.Sprintf("projects/%s/secrets/%s", s.projectID, key), + } + err = client.DeleteSecret(ctx, req) + Expect(err).ToNot(HaveOccurred()) +} + +func (s *gcpProvider) BeforeEach() { + By("creating a gcp secret") + gcpCreds := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "provider-secret", + Namespace: s.framework.Namespace.Name, + }, + StringData: map[string]string{ + "secret-access-credentials": s.credentials, + }, + } + err := s.framework.CRClient.Create(context.Background(), gcpCreds) + Expect(err).ToNot(HaveOccurred()) + + By("creating an secret store for vault") + secretStore := &esv1alpha1.SecretStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: s.framework.Namespace.Name, + Namespace: s.framework.Namespace.Name, + }, + Spec: esv1alpha1.SecretStoreSpec{ + Provider: &esv1alpha1.SecretStoreProvider{ + GCPSM: &esv1alpha1.GCPSMProvider{ + ProjectID: s.projectID, + Auth: esv1alpha1.GCPSMAuth{ + SecretRef: esv1alpha1.GCPSMAuthSecretRef{ + SecretAccessKey: esmeta.SecretKeySelector{ + Name: "provider-secret", + Key: "secret-access-credentials", + }, + }, + }, + }, + }, + }, + } + err = s.framework.CRClient.Create(context.Background(), secretStore) + Expect(err).ToNot(HaveOccurred()) +} diff --git a/e2e/suite/gcp/util.go b/e2e/suite/gcp/util.go deleted file mode 100644 index 8ecc58b6a..000000000 --- a/e2e/suite/gcp/util.go +++ /dev/null @@ -1,101 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -limitations under the License. -*/ -package gcp - -import ( - "context" - "fmt" - - secretmanager "cloud.google.com/go/secretmanager/apiv1" - "golang.org/x/oauth2/google" - "google.golang.org/api/option" - secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1" - - gcpsm "github.com/external-secrets/external-secrets/pkg/provider/gcp/secretmanager" -) - -// CreateAWSSecretsManagerSecret creates a sm secret with the given value. -func createGCPSecretsManagerSecret(projectID, secretName, secretValue string, credentials []byte) (*secretmanagerpb.Secret, error) { - ctx := context.Background() - - config, err := google.JWTConfigFromJSON(credentials, gcpsm.CloudPlatformRole) - if err != nil { - return nil, fmt.Errorf("unable to procces JSON credentials: %w", err) - } - ts := config.TokenSource(ctx) - - client, err := secretmanager.NewClient(ctx, option.WithTokenSource(ts)) - if err != nil { - return nil, fmt.Errorf("failed to setup client: %w", err) - } - defer client.Close() - // Create the request to create the secret. - createSecretReq := &secretmanagerpb.CreateSecretRequest{ - Parent: fmt.Sprintf("projects/%s", projectID), - SecretId: secretName, - Secret: &secretmanagerpb.Secret{ - Replication: &secretmanagerpb.Replication{ - Replication: &secretmanagerpb.Replication_Automatic_{ - Automatic: &secretmanagerpb.Replication_Automatic{}, - }, - }, - }, - } - secret, err := client.CreateSecret(ctx, createSecretReq) - if err != nil { - return nil, fmt.Errorf("failed to create secret: %w", err) - } - // Declare the payload to store. - payload := []byte(secretValue) - // Build the request. - addSecretVersionReq := &secretmanagerpb.AddSecretVersionRequest{ - Parent: secret.Name, - Payload: &secretmanagerpb.SecretPayload{ - Data: payload, - }, - } - // Call the API. - _, err = client.AddSecretVersion(ctx, addSecretVersionReq) - if err != nil { - return nil, fmt.Errorf("failed to add secret version: %w", err) - } - - return secret, err -} - -// deleteSecret deletes the secret with the given name and all of its versions. -func deleteGCPSecretsManagerSecret(secretName string, credentials []byte) error { - ctx := context.Background() - config, err := google.JWTConfigFromJSON(credentials, gcpsm.CloudPlatformRole) - if err != nil { - return fmt.Errorf("unable to procces JSON credentials: %w", err) - } - ts := config.TokenSource(ctx) - - client, err := secretmanager.NewClient(ctx, option.WithTokenSource(ts)) - if err != nil { - return fmt.Errorf("failed to setup client: %w", err) - } - defer client.Close() - - // Build the request. - req := &secretmanagerpb.DeleteSecretRequest{ - Name: secretName, - } - - // Call the API. - if err := client.DeleteSecret(ctx, req); err != nil { - return fmt.Errorf("failed to delete secret: %w", err) - } - return nil -} diff --git a/e2e/suite/vault/provider.go b/e2e/suite/vault/provider.go new file mode 100644 index 000000000..c36b5c97a --- /dev/null +++ b/e2e/suite/vault/provider.go @@ -0,0 +1,111 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package vault + +import ( + "context" + "fmt" + "net/http" + + vault "github.com/hashicorp/vault/api" + + //nolint + . "github.com/onsi/ginkgo" + + //nolint + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" + esmeta "github.com/external-secrets/external-secrets/apis/meta/v1" + "github.com/external-secrets/external-secrets/e2e/framework" +) + +type vaultProvider struct { + url string + token string + client *vault.Client + framework *framework.Framework +} + +func newVaultProvider(f *framework.Framework, url, token string) *vaultProvider { + vc, err := vault.NewClient(&vault.Config{ + Address: url, + }) + Expect(err).ToNot(HaveOccurred()) + vc.SetToken(token) + + prov := &vaultProvider{ + framework: f, + url: url, + token: token, + client: vc, + } + BeforeEach(prov.BeforeEach) + return prov +} + +func (s *vaultProvider) CreateSecret(key, val string) { + req := s.client.NewRequest(http.MethodPost, fmt.Sprintf("/v1/secret/data/%s", key)) + req.BodyBytes = []byte(fmt.Sprintf(`{"data": %s}`, val)) + _, err := s.client.RawRequestWithContext(context.Background(), req) + Expect(err).ToNot(HaveOccurred()) +} + +func (s *vaultProvider) DeleteSecret(key string) { + req := s.client.NewRequest(http.MethodDelete, fmt.Sprintf("/v1/secret/data/%s", key)) + _, err := s.client.RawRequestWithContext(context.Background(), req) + Expect(err).ToNot(HaveOccurred()) +} + +func (s *vaultProvider) BeforeEach() { + By("creating a vault secret") + vaultCreds := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "provider-secret", + Namespace: s.framework.Namespace.Name, + }, + StringData: map[string]string{ + "token": s.token, // vault dev-mode default token + }, + } + err := s.framework.CRClient.Create(context.Background(), vaultCreds) + Expect(err).ToNot(HaveOccurred()) + + By("creating an secret store for vault") + secretStore := &esv1alpha1.SecretStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: s.framework.Namespace.Name, + Namespace: s.framework.Namespace.Name, + }, + Spec: esv1alpha1.SecretStoreSpec{ + Provider: &esv1alpha1.SecretStoreProvider{ + Vault: &esv1alpha1.VaultProvider{ + Version: esv1alpha1.VaultKVStoreV2, + Path: "secret", + Server: s.url, + Auth: esv1alpha1.VaultAuth{ + TokenSecretRef: &esmeta.SecretKeySelector{ + Name: "provider-secret", + Key: "token", + }, + }, + }, + }, + }, + } + err = s.framework.CRClient.Create(context.Background(), secretStore) + Expect(err).ToNot(HaveOccurred()) +} diff --git a/e2e/suite/vault/vault.go b/e2e/suite/vault/vault.go index b8fcf2ce9..bf805857f 100644 --- a/e2e/suite/vault/vault.go +++ b/e2e/suite/vault/vault.go @@ -13,116 +13,24 @@ limitations under the License. package vault import ( - "context" - "fmt" - "net/http" - - vault "github.com/hashicorp/vault/api" // nolint . "github.com/onsi/ginkgo" // nolint - . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + . "github.com/onsi/ginkgo/extensions/table" - esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" - esmeta "github.com/external-secrets/external-secrets/apis/meta/v1" "github.com/external-secrets/external-secrets/e2e/framework" + "github.com/external-secrets/external-secrets/e2e/suite/common" ) var _ = Describe("[vault] ", func() { f := framework.New("eso-vault") - var secretStore *esv1alpha1.SecretStore - BeforeEach(func() { - By("creating an secret store for vault") - vaultCreds := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: f.Namespace.Name, - Namespace: f.Namespace.Name, - }, - StringData: map[string]string{ - "token": "root", // vault dev-mode default token - }, - } - err := f.CRClient.Create(context.Background(), vaultCreds) - Expect(err).ToNot(HaveOccurred()) - secretStore = &esv1alpha1.SecretStore{ - ObjectMeta: metav1.ObjectMeta{ - Name: f.Namespace.Name, - Namespace: f.Namespace.Name, - }, - Spec: esv1alpha1.SecretStoreSpec{ - Provider: &esv1alpha1.SecretStoreProvider{ - Vault: &esv1alpha1.VaultProvider{ - Version: esv1alpha1.VaultKVStoreV2, - Path: "secret", - Server: "http://vault.default:8200", - Auth: esv1alpha1.VaultAuth{ - TokenSecretRef: &esmeta.SecretKeySelector{ - Name: f.Namespace.Name, - Key: "token", - }, - }, - }, - }, - }, - } - err = f.CRClient.Create(context.Background(), secretStore) - Expect(err).ToNot(HaveOccurred()) - }) - - It("should sync secrets", func() { - secretKey := fmt.Sprintf("%s-%s", f.Namespace.Name, "one") - secretProp := "example" - secretValue := "bar" - targetSecret := "target-secret" - - By("creating a vault secret") - vc, err := vault.NewClient(&vault.Config{ - Address: "http://vault.default:8200", - }) - Expect(err).ToNot(HaveOccurred()) - vc.SetToken("root") // dev-mode default token - req := vc.NewRequest(http.MethodPost, fmt.Sprintf("/v1/secret/data/%s", secretKey)) - err = req.SetJSONBody(map[string]interface{}{ - "data": map[string]string{ - secretProp: secretValue, - }, - }) - Expect(err).ToNot(HaveOccurred()) - _, err = vc.RawRequestWithContext(context.Background(), req) - Expect(err).ToNot(HaveOccurred()) - - By("creating ExternalSecret") - err = f.CRClient.Create(context.Background(), &esv1alpha1.ExternalSecret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "simple-sync", - Namespace: f.Namespace.Name, - }, - Spec: esv1alpha1.ExternalSecretSpec{ - SecretStoreRef: esv1alpha1.SecretStoreRef{ - Name: secretStore.Name, - }, - Target: esv1alpha1.ExternalSecretTarget{ - Name: targetSecret, - }, - Data: []esv1alpha1.ExternalSecretData{ - { - SecretKey: secretKey, - RemoteRef: esv1alpha1.ExternalSecretDataRemoteRef{ - Key: secretKey, - Property: secretProp, - }, - }, - }, - }, - }) - Expect(err).ToNot(HaveOccurred()) - _, err = f.WaitForSecretValue(f.Namespace.Name, targetSecret, map[string][]byte{ - secretKey: []byte(secretValue), - }) - Expect(err).ToNot(HaveOccurred()) - }) + DescribeTable("sync secrets", + framework.TableFunc(f, + newVaultProvider(f, "http://vault.default:8200", "root")), + Entry(common.JSONDataFromSync(f)), + Entry(common.JSONDataWithProperty(f)), + Entry(common.JSONDataWithTemplate(f)), + ) }) diff --git a/pkg/provider/azure/keyvault/keyvault.go b/pkg/provider/azure/keyvault/keyvault.go index bd7888367..2152b6679 100644 --- a/pkg/provider/azure/keyvault/keyvault.go +++ b/pkg/provider/azure/keyvault/keyvault.go @@ -23,6 +23,7 @@ import ( "github.com/Azure/azure-sdk-for-go/profiles/latest/keyvault/keyvault" kvauth "github.com/Azure/go-autorest/autorest/azure/auth" + "github.com/tidwall/gjson" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -110,7 +111,14 @@ func (a *Azure) GetSecret(ctx context.Context, ref esv1alpha1.ExternalSecretData if err != nil { return nil, err } - return []byte(*secretResp.Value), nil + if ref.Property == "" { + return []byte(*secretResp.Value), nil + } + res := gjson.Get(*secretResp.Value, ref.Property) + if !res.Exists() { + return nil, fmt.Errorf("property %s does not exist in key %s", ref.Property, ref.Key) + } + return []byte(res.String()), err case "cert": // returns a CertBundle. We return CER contents of x509 certificate // see: https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/services/keyvault/v7.0/keyvault#CertificateBundle diff --git a/pkg/provider/gcp/secretmanager/secretsmanager.go b/pkg/provider/gcp/secretmanager/secretsmanager.go index a1ecd453f..e5072b5b3 100644 --- a/pkg/provider/gcp/secretmanager/secretsmanager.go +++ b/pkg/provider/gcp/secretmanager/secretsmanager.go @@ -167,7 +167,9 @@ func (sm *ProviderGCP) GetSecret(ctx context.Context, ref esv1alpha1.ExternalSec } val := gjson.Get(payload, ref.Property) - + if !val.Exists() { + return nil, fmt.Errorf("key %s does not exist in secret %s", ref.Property, ref.Key) + } return []byte(val.String()), nil }