Merge pull request #217 from ContainerSolutions/gcp_property

feat: add property feature to gcp

.github/codecov.yml

@@ -1,2 +1,5 @@
 ignore:
 - pkg/provider/**/fake
+coverage:
+  round: down
+  precision: 2

docs/provider-google-secrets-manager.md

@@ -31,5 +31,5 @@ To create a kubernetes secret from the GCP Secret Manager secret a `Kind=Externa
 
 The operator will fetch the GCP Secret Manager secret and inject it as a `Kind=Secret`
 ```
-kubectl get secret secret-to-be-created -n <namespace> | -o jsonpath='{.data.example-externalsecret-key}' | base64 -d
+kubectl get secret secret-to-be-created -n <namespace> | -o jsonpath='{.data.dev-secret-test}' | base64 -d
 ```

e2e/suite/aws/secretsmanager.go

@@ -134,7 +134,7 @@ var _ = Describe("[aws] ", func() {
 	})
 
 	It("should sync secrets with dataFrom", func() {
-		By("creating a GCP SM Secret")
+		By("creating a AWS SM Secret")
 		secretKey1 := fmt.Sprintf("%s-%s", f.Namespace.Name, "one")
 		targetSecretKey1 := "name"
 		targetSecretValue1 := "great-name"
@@ -174,7 +174,7 @@ var _ = Describe("[aws] ", func() {
 	})
 
 	It("should sync secrets and get inner keys", func() {
-		By("creating a GCP SM Secret")
+		By("creating a AWS SM Secret")
 		secretKey1 := fmt.Sprintf("%s-%s", f.Namespace.Name, "one")
 		targetSecretKey1 := "firstname"
 		targetSecretValue1 := "Tom"

e2e/suite/gcp/gcp.go

@@ -29,6 +29,10 @@ import (
 	"github.com/external-secrets/external-secrets/e2e/framework"
 )
 
+const (
+	targetSecret = "target-secret"
+)
+
 var _ = Describe("[gcp] ", func() {
 	f := framework.New("eso-gcp")
 	var secretStore *esv1alpha1.SecretStore
@@ -77,7 +81,6 @@ var _ = Describe("[gcp] ", func() {
 		By("creating a GCP SM Secret")
 		secretKey1 := fmt.Sprintf("%s-%s", f.Namespace.Name, "one")
 		secretValue := "great-value-test"
-		targetSecret := "target-secret"
 		secret, err := createGCPSecretsManagerSecret(
 			projectID,
 			secretKey1, secretValue, []byte(credentials))
@@ -123,7 +126,6 @@ var _ = Describe("[gcp] ", func() {
 		targetSecretKey2 := "surname"
 		targetSecretValue2 := "great-surname"
 		secretValue := fmt.Sprintf("{ \"%s\": \"%s\", \"%s\": \"%s\" }", targetSecretKey1, targetSecretValue1, targetSecretKey2, targetSecretValue2)
-		targetSecret := "target-secret"
 		secret, err := createGCPSecretsManagerSecret(
 			projectID,
 			secretKey1, secretValue, []byte(credentials))
@@ -159,4 +161,67 @@ var _ = Describe("[gcp] ", func() {
 		Expect(err).ToNot(HaveOccurred())
 	})
 
+	It("should sync secrets and get inner keys", func() {
+		By("creating a GCP SM Secret")
+		secretKey1 := fmt.Sprintf("%s-%s", f.Namespace.Name, "one")
+		targetSecretKey1 := "firstname"
+		targetSecretValue1 := "Tom"
+		targetSecretKey2 := "first_friend"
+		targetSecretValue2 := "Roger"
+		secretValue := fmt.Sprintf(
+			`{
+				"name": {"first": "%s", "last": "Anderson"},
+				"friends": 
+				[ 
+					{"first": "Dale", "last": "Murphy"}, 
+					{"first": "%s", "last": "Craig"}, 
+					{"first": "Jane", "last": "Murphy"} 
+				]
+			}`, targetSecretValue1, targetSecretValue2)
+		secret, err := createGCPSecretsManagerSecret(
+			projectID,
+			secretKey1, secretValue, []byte(credentials))
+		Expect(err).ToNot(HaveOccurred())
+		err = f.CRClient.Create(context.Background(), &esv1alpha1.ExternalSecret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "datafrom-sync",
+				Namespace: f.Namespace.Name,
+			},
+			Spec: esv1alpha1.ExternalSecretSpec{
+				SecretStoreRef: esv1alpha1.SecretStoreRef{
+					Name: f.Namespace.Name,
+				},
+				Target: esv1alpha1.ExternalSecretTarget{
+					Name: targetSecret,
+				},
+				Data: []esv1alpha1.ExternalSecretData{
+					{
+						SecretKey: targetSecretKey1,
+						RemoteRef: esv1alpha1.ExternalSecretDataRemoteRef{
+							Key:      secretKey1,
+							Property: "name.first",
+						},
+					},
+					{
+						SecretKey: targetSecretKey2,
+						RemoteRef: esv1alpha1.ExternalSecretDataRemoteRef{
+							Key:      secretKey1,
+							Property: "friends.1.first",
+						},
+					},
+				},
+			},
+		})
+		Expect(err).ToNot(HaveOccurred())
+
+		_, err = f.WaitForSecretValue(f.Namespace.Name, targetSecret, map[string][]byte{
+			targetSecretKey1: []byte(targetSecretValue1),
+			targetSecretKey2: []byte(targetSecretValue2),
+		})
+		Expect(err).ToNot(HaveOccurred())
+
+		err = deleteGCPSecretsManagerSecret(secret.Name, []byte(credentials))
+		Expect(err).ToNot(HaveOccurred())
+	})
+
 })

pkg/controllers/externalsecret/externalsecret_controller.go

@@ -294,6 +294,11 @@ func (r *Reconciler) getProviderSecretData(ctx context.Context, providerClient p
 		providerData[secretRef.SecretKey] = secretData
 	}
 
+	err := providerClient.Close()
+	if err != nil {
+		return nil, fmt.Errorf("error closing the connection: %w", err)
+	}
+
 	return providerData, nil
 }
 

pkg/provider/aws/parameterstore/parameterstore.go

@@ -89,3 +89,7 @@ func (pm *ParameterStore) GetSecretMap(ctx context.Context, ref esv1alpha1.Exter
 	}
 	return secretData, nil
 }
+
+func (pm *ParameterStore) Close() error {
+	return nil
+}

pkg/provider/aws/secretsmanager/secretsmanager.go

@@ -102,3 +102,7 @@ func (sm *SecretsManager) GetSecretMap(ctx context.Context, ref esv1alpha1.Exter
 	}
 	return secretData, nil
 }
+
+func (sm *SecretsManager) Close() error {
+	return nil
+}

pkg/provider/azure/keyvault/keyvault.go

@@ -225,3 +225,7 @@ func (a *Azure) secretKeyRef(ctx context.Context, namespace string, secretRef sm
 	value := strings.TrimSpace(string(keyBytes))
 	return value, nil
 }
+
+func (a *Azure) Close() error {
+	return nil
+}

pkg/provider/fake/fake.go

@@ -74,6 +74,9 @@ func (v *Client) WithGetSecret(secData []byte, err error) *Client {
 func (v *Client) GetSecretMap(ctx context.Context, ref esv1alpha1.ExternalSecretDataRemoteRef) (map[string][]byte, error) {
 	return v.GetSecretMapFn(ctx, ref)
 }
+func (v *Client) Close() error {
+	return nil
+}
 
 // WithGetSecretMap wraps the secret data map returned by this fake provider.
 func (v *Client) WithGetSecretMap(secData map[string][]byte, err error) *Client {

pkg/provider/gcp/secretmanager/secretsmanager.go

@@ -20,6 +20,7 @@ import (
 
 	secretmanager "cloud.google.com/go/secretmanager/apiv1"
 	"github.com/googleapis/gax-go"
+	"github.com/tidwall/gjson"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/option"
 	secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1"
@@ -38,6 +39,7 @@ const (
 
 	errGCPSMStore                             = "received invalid GCPSM SecretStore resource"
 	errGCPSMCredSecretName                    = "invalid GCPSM SecretStore resource: missing GCP Secret Access Key"
+	errClientClose                            = "unable to close SecretManager client: %w"
 	errInvalidClusterStoreMissingSAKNamespace = "invalid ClusterSecretStore: missing GCP SecretAccessKey Namespace"
 	errFetchSAKSecret                         = "could not fetch SecretAccessKey secret: %w"
 	errMissingSAK                             = "missing SecretAccessKey"
@@ -45,7 +47,6 @@ const (
 	errUnableCreateGCPSMClient                = "failed to create GCP secretmanager client: %w"
 	errUninitalizedGCPProvider                = "provider GCP is not initialized"
 	errClientGetSecretAccess                  = "unable to access Secret from SecretManager Client: %w"
-	errClientClose                            = "unable to close SecretManager client: %w"
 	errJSONSecretUnmarshal                    = "unable to unmarshal secret: %w"
 )
 
@@ -153,12 +154,21 @@ func (sm *ProviderGCP) GetSecret(ctx context.Context, ref esv1alpha1.ExternalSec
 		return nil, fmt.Errorf(errClientGetSecretAccess, err)
 	}
 
-	err = sm.SecretManagerClient.Close()
-	if err != nil {
-		return nil, fmt.Errorf(errClientClose, err)
+	if ref.Property == "" {
+		if result.Payload.Data != nil {
+			return result.Payload.Data, nil
+		}
+		return nil, fmt.Errorf("invalid secret received. no secret string for key: %s", ref.Key)
 	}
 
-	return result.Payload.Data, nil
+	var payload string
+	if result.Payload.Data != nil {
+		payload = string(result.Payload.Data)
+	}
+
+	val := gjson.Get(payload, ref.Property)
+
+	return []byte(val.String()), nil
 }
 
 // GetSecretMap returns multiple k/v pairs from the provider.
@@ -186,6 +196,14 @@ func (sm *ProviderGCP) GetSecretMap(ctx context.Context, ref esv1alpha1.External
 	return secretData, nil
 }
 
+func (sm *ProviderGCP) Close() error {
+	err := sm.SecretManagerClient.Close()
+	if err != nil {
+		return fmt.Errorf(errClientClose, err)
+	}
+	return nil
+}
+
 func init() {
 	schema.Register(&ProviderGCP{}, &esv1alpha1.SecretStoreProvider{
 		GCPSM: &esv1alpha1.GCPSMProvider{},

pkg/provider/gcp/secretmanager/secretsmanager_test.go

@@ -103,6 +103,26 @@ func TestSecretManagerGetSecret(t *testing.T) {
 		smtc.expectedSecret = "testtesttest"
 	}
 
+	// good case: ref with
+	setCustomRef := func(smtc *secretManagerTestCase) {
+		smtc.ref = &esv1alpha1.ExternalSecretDataRemoteRef{
+			Key:      "/baz",
+			Version:  "default",
+			Property: "name.first",
+		}
+		smtc.apiInput.Name = "projects/default/secrets//baz/versions/default"
+		smtc.apiOutput.Payload.Data = []byte(
+			`{
+			"name": {"first": "Tom", "last": "Anderson"},
+			"friends": [
+				{"first": "Dale", "last": "Murphy"},
+				{"first": "Roger", "last": "Craig"},
+				{"first": "Jane", "last": "Murphy"}
+			]
+        }`)
+		smtc.expectedSecret = "Tom"
+	}
+
 	// good case: custom version set
 	setCustomVersion := func(smtc *secretManagerTestCase) {
 		smtc.ref.Version = "1234"
@@ -116,6 +136,7 @@ func TestSecretManagerGetSecret(t *testing.T) {
 		makeValidSecretManagerTestCaseCustom(setSecretString),
 		makeValidSecretManagerTestCaseCustom(setCustomVersion),
 		makeValidSecretManagerTestCaseCustom(setAPIErr),
+		makeValidSecretManagerTestCaseCustom(setCustomRef),
 	}
 
 	sm := ProviderGCP{}

pkg/provider/provider.go

@@ -35,4 +35,5 @@ type SecretsClient interface {
 
 	// GetSecretMap returns multiple k/v pairs from the provider
 	GetSecretMap(ctx context.Context, ref esv1alpha1.ExternalSecretDataRemoteRef) (map[string][]byte, error)
+	Close() error
 }

pkg/provider/schema/schema_test.go

@@ -41,6 +41,10 @@ func (p *PP) GetSecretMap(ctx context.Context, ref esv1alpha1.ExternalSecretData
 	return map[string][]byte{}, nil
 }
 
+func (p *PP) Close() error {
+	return nil
+}
+
 // TestRegister tests if the Register function
 // (1) panics if it tries to register something invalid
 // (2) stores the correct provider.

pkg/provider/vault/vault.go

@@ -149,6 +149,10 @@ func (v *client) GetSecretMap(ctx context.Context, ref esv1alpha1.ExternalSecret
 	return v.readSecret(ctx, ref.Key, ref.Version)
 }
 
+func (v *client) Close() error {
+	return nil
+}
+
 func (v *client) readSecret(ctx context.Context, path, version string) (map[string][]byte, error) {
 	kvPath := v.store.Path