diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f66d701b0..ee0f0e01e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,7 +176,7 @@ jobs: make test - name: Publish Unit Test Coverage - uses: codecov/codecov-action@v2.0.2 + uses: codecov/codecov-action@v2.0.3 with: flags: unittests file: ./cover.out diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e151c841e..a4c4bae37 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -151,7 +151,7 @@ jobs: make test.e2e # Update check run called "integration-fork" - - uses: actions/github-script@v1 + - uses: actions/github-script@v4.1 id: update-check-run if: ${{ always() }} env: diff --git a/.github/workflows/ok-to-test.yml b/.github/workflows/ok-to-test.yml index 3653b9508..7b9d57e7a 100644 --- a/.github/workflows/ok-to-test.yml +++ b/.github/workflows/ok-to-test.yml @@ -23,7 +23,7 @@ jobs: private_key: ${{ secrets.PRIVATE_KEY }} - name: Slash Command Dispatch - uses: peter-evans/slash-command-dispatch@v2.2.1 + uses: peter-evans/slash-command-dispatch@v2.3.0 env: TOKEN: ${{ steps.generate_token.outputs.token }} with: diff --git a/Dockerfile b/Dockerfile index d9ab2d93b..5a1ca119a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.14.0 +FROM alpine:3.14.2 ARG TARGETOS ARG TARGETARCH COPY bin/external-secrets-${TARGETOS}-${TARGETARCH} /bin/external-secrets diff --git a/README.md b/README.md index 4db54e1c4..c3a89c451 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,10 @@ Multiple people and organizations are joining efforts to create a single Externa - [Google Cloud Secrets Manager](https://external-secrets.io/provider-google-secrets-manager/) - [Azure Key Vault](https://external-secrets.io/provider-azure-key-vault/) - [IBM Cloud Secrets Manager](https://external-secrets.io/provider-ibm-secrets-manager/) +- [Yandex Lockbox](https://external-secrets.io/provider-yandex-lockbox/) +- [Gitlab Project Variables](https://external-secrets.io/provider-gitlab-project-variables/) +- [Alibaba Cloud KMS](https://www.alibabacloud.com/product/kms) (Docs still missing, PRs welcomed!) +- [Oracle Vault]( https://external-secrets.io/provider-oracle-vault) ## Stability and Support Level @@ -35,6 +39,11 @@ Multiple people and organizations are joining efforts to create a single Externa | ------------------------------------------------------------------- | :-------: | :----------------------------------------: | | [Azure KV](https://external-secrets.io/provider-azure-key-vault/) | alpha | @ahmedmus-1A @asnowfix @ncourbet-1A @1A-mj | | [IBM SM](https://external-secrets.io/provider-ibm-secrets-manager/) | alpha | @knelasevero @sebagomez @ricardoptcosta | +| [Yandex Lockbox](https://external-secrets.io/provider-yandex-lockbox/) | alpha | @AndreyZamyslov @knelasevero | +| [Gitlab Project Variables](https://external-secrets.io/provider-gitlab-project-variables/) | alpha | @Jabray5 | +| Alibaba Cloud KMS | alpha | @ElsaChelala | +| [Oracle Vault]( https://external-secrets.io/provider-oracle-vault) | alpha | @KianTigger | + ## Documentation diff --git a/apis/externalsecrets/v1alpha1/externalsecret_types.go b/apis/externalsecrets/v1alpha1/externalsecret_types.go index e192daba6..3395012e8 100644 --- a/apis/externalsecrets/v1alpha1/externalsecret_types.go +++ b/apis/externalsecrets/v1alpha1/externalsecret_types.go @@ -211,6 +211,11 @@ type ExternalSecret struct { Status ExternalSecretStatus `json:"status,omitempty"` } +const ( + // AnnotationDataHash is used to ensure consistency. + AnnotationDataHash = "reconcile.external-secrets.io/data-hash" +) + // +kubebuilder:object:root=true // ExternalSecretList contains a list of ExternalSecret resources. diff --git a/apis/externalsecrets/v1alpha1/secretstore_alibaba_types.go b/apis/externalsecrets/v1alpha1/secretstore_alibaba_types.go new file mode 100644 index 000000000..7b60ffe1f --- /dev/null +++ b/apis/externalsecrets/v1alpha1/secretstore_alibaba_types.go @@ -0,0 +1,41 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + esmeta "github.com/external-secrets/external-secrets/apis/meta/v1" +) + +// AlibabaAuth contains a secretRef for credentials. +type AlibabaAuth struct { + SecretRef AlibabaAuthSecretRef `json:"secretRef"` +} + +// AlibabaAuthSecretRef holds secret references for Alibaba credentials. +type AlibabaAuthSecretRef struct { + // The AccessKeyID is used for authentication + AccessKeyID esmeta.SecretKeySelector `json:"accessKeyIDSecretRef"` + // The AccessKeySecret is used for authentication + AccessKeySecret esmeta.SecretKeySelector `json:"accessKeySecretSecretRef"` +} + +// AlibabaProvider configures a store to sync secrets using the Alibaba Secret Manager provider. +type AlibabaProvider struct { + Auth *AlibabaAuth `json:"auth"` + // +optional + Endpoint string `json:"endpoint"` + // Alibaba Region to be used for the provider + RegionID string `json:"regionID"` +} diff --git a/apis/externalsecrets/v1alpha1/secretstore_gcpsm_types.go b/apis/externalsecrets/v1alpha1/secretstore_gcpsm_types.go index e410db65c..709914880 100644 --- a/apis/externalsecrets/v1alpha1/secretstore_gcpsm_types.go +++ b/apis/externalsecrets/v1alpha1/secretstore_gcpsm_types.go @@ -31,7 +31,8 @@ type GCPSMAuthSecretRef struct { // GCPSMProvider Configures a store to sync secrets using the GCP Secret Manager provider. type GCPSMProvider struct { // Auth defines the information necessary to authenticate against GCP - Auth GCPSMAuth `json:"auth"` + // +optional + Auth GCPSMAuth `json:"auth,omitempty"` // ProjectID project where secret is located ProjectID string `json:"projectID,omitempty"` diff --git a/apis/externalsecrets/v1alpha1/secretstore_gitlab_types.go b/apis/externalsecrets/v1alpha1/secretstore_gitlab_types.go new file mode 100644 index 000000000..6d350875c --- /dev/null +++ b/apis/externalsecrets/v1alpha1/secretstore_gitlab_types.go @@ -0,0 +1,40 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + esmeta "github.com/external-secrets/external-secrets/apis/meta/v1" +) + +// Configures a store to sync secrets with a GitLab instance. +type GitlabProvider struct { + // URL configures the GitLab instance URL. Defaults to https://gitlab.com/. + URL string `json:"url,omitempty"` + + // Auth configures how secret-manager authenticates with a GitLab instance. + Auth GitlabAuth `json:"auth"` + + // ProjectID specifies a project where secrets are located. + ProjectID string `json:"projectID,omitempty"` +} + +type GitlabAuth struct { + SecretRef GitlabSecretRef `json:"SecretRef"` +} + +type GitlabSecretRef struct { + // AccessToken is used for authentication. + AccessToken esmeta.SecretKeySelector `json:"accessToken,omitempty"` +} diff --git a/apis/externalsecrets/v1alpha1/secretstore_oracle_types.go b/apis/externalsecrets/v1alpha1/secretstore_oracle_types.go new file mode 100644 index 000000000..73db29e25 --- /dev/null +++ b/apis/externalsecrets/v1alpha1/secretstore_oracle_types.go @@ -0,0 +1,46 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + esmeta "github.com/external-secrets/external-secrets/apis/meta/v1" +) + +// Configures an store to sync secrets using a Oracle Vault +// backend. +type OracleProvider struct { + // Auth configures how secret-manager authenticates with the Oracle Vault. + Auth OracleAuth `json:"auth"` + + // User is an access OCID specific to the account. + User string `json:"user,omitempty"` + + // projectID is an access token specific to the secret. + Tenancy string `json:"tenancy,omitempty"` + + // projectID is an access token specific to the secret. + Region string `json:"region,omitempty"` +} + +type OracleAuth struct { + // SecretRef to pass through sensitive information. + SecretRef OracleSecretRef `json:"secretRef"` +} + +type OracleSecretRef struct { + // The Access Token is used for authentication + PrivateKey esmeta.SecretKeySelector `json:"privatekey,omitempty"` + + // projectID is an access token specific to the secret. + Fingerprint esmeta.SecretKeySelector `json:"fingerprint,omitempty"` +} diff --git a/apis/externalsecrets/v1alpha1/secretstore_types.go b/apis/externalsecrets/v1alpha1/secretstore_types.go index 607f4fbf5..adf136dae 100644 --- a/apis/externalsecrets/v1alpha1/secretstore_types.go +++ b/apis/externalsecrets/v1alpha1/secretstore_types.go @@ -50,9 +50,25 @@ type SecretStoreProvider struct { // +optional GCPSM *GCPSMProvider `json:"gcpsm,omitempty"` + // Oracle configures this store to sync secrets using Oracle Vault provider + // +optional + Oracle *OracleProvider `json:"oracle,omitempty"` + // IBM configures this store to sync secrets using IBM Cloud provider // +optional IBM *IBMProvider `json:"ibm,omitempty"` + + // YandexLockbox configures this store to sync secrets using Yandex Lockbox provider + // +optional + YandexLockbox *YandexLockboxProvider `json:"yandexlockbox,omitempty"` + + // GItlab configures this store to sync secrets using Gitlab Variables provider + // +optional + Gitlab *GitlabProvider `json:"gitlab,omitempty"` + + // Alibaba configures this store to sync secrets using Alibaba Cloud provider + // +optional + Alibaba *AlibabaProvider `json:"alibaba,omitempty"` } type SecretStoreConditionType string diff --git a/apis/externalsecrets/v1alpha1/secretstore_yandexlockbox_types.go b/apis/externalsecrets/v1alpha1/secretstore_yandexlockbox_types.go new file mode 100644 index 000000000..d4eabc27d --- /dev/null +++ b/apis/externalsecrets/v1alpha1/secretstore_yandexlockbox_types.go @@ -0,0 +1,35 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + esmeta "github.com/external-secrets/external-secrets/apis/meta/v1" +) + +type YandexLockboxAuth struct { + // The authorized key used for authentication + // +optional + AuthorizedKey esmeta.SecretKeySelector `json:"authorizedKeySecretRef,omitempty"` +} + +// YandexLockboxProvider Configures a store to sync secrets using the Yandex Lockbox provider. +type YandexLockboxProvider struct { + // Yandex.Cloud API endpoint (e.g. 'api.cloud.yandex.net:443') + // +optional + APIEndpoint string `json:"apiEndpoint,omitempty"` + + // Auth defines the information necessary to authenticate against Yandex Lockbox + Auth YandexLockboxAuth `json:"auth"` +} diff --git a/apis/externalsecrets/v1alpha1/zz_generated.deepcopy.go b/apis/externalsecrets/v1alpha1/zz_generated.deepcopy.go index 4a0e8d6b7..08d28a16a 100644 --- a/apis/externalsecrets/v1alpha1/zz_generated.deepcopy.go +++ b/apis/externalsecrets/v1alpha1/zz_generated.deepcopy.go @@ -102,6 +102,59 @@ func (in *AWSProvider) DeepCopy() *AWSProvider { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AlibabaAuth) DeepCopyInto(out *AlibabaAuth) { + *out = *in + in.SecretRef.DeepCopyInto(&out.SecretRef) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AlibabaAuth. +func (in *AlibabaAuth) DeepCopy() *AlibabaAuth { + if in == nil { + return nil + } + out := new(AlibabaAuth) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AlibabaAuthSecretRef) DeepCopyInto(out *AlibabaAuthSecretRef) { + *out = *in + in.AccessKeyID.DeepCopyInto(&out.AccessKeyID) + in.AccessKeySecret.DeepCopyInto(&out.AccessKeySecret) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AlibabaAuthSecretRef. +func (in *AlibabaAuthSecretRef) DeepCopy() *AlibabaAuthSecretRef { + if in == nil { + return nil + } + out := new(AlibabaAuthSecretRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AlibabaProvider) DeepCopyInto(out *AlibabaProvider) { + *out = *in + if in.Auth != nil { + in, out := &in.Auth, &out.Auth + *out = new(AlibabaAuth) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AlibabaProvider. +func (in *AlibabaProvider) DeepCopy() *AlibabaProvider { + if in == nil { + return nil + } + out := new(AlibabaProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureKVAuth) DeepCopyInto(out *AzureKVAuth) { *out = *in @@ -504,6 +557,54 @@ func (in *GCPSMProvider) DeepCopy() *GCPSMProvider { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GitlabAuth) DeepCopyInto(out *GitlabAuth) { + *out = *in + in.SecretRef.DeepCopyInto(&out.SecretRef) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitlabAuth. +func (in *GitlabAuth) DeepCopy() *GitlabAuth { + if in == nil { + return nil + } + out := new(GitlabAuth) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GitlabProvider) DeepCopyInto(out *GitlabProvider) { + *out = *in + in.Auth.DeepCopyInto(&out.Auth) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitlabProvider. +func (in *GitlabProvider) DeepCopy() *GitlabProvider { + if in == nil { + return nil + } + out := new(GitlabProvider) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GitlabSecretRef) DeepCopyInto(out *GitlabSecretRef) { + *out = *in + in.AccessToken.DeepCopyInto(&out.AccessToken) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitlabSecretRef. +func (in *GitlabSecretRef) DeepCopy() *GitlabSecretRef { + if in == nil { + return nil + } + out := new(GitlabSecretRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IBMAuth) DeepCopyInto(out *IBMAuth) { *out = *in @@ -557,6 +658,55 @@ func (in *IBMProvider) DeepCopy() *IBMProvider { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OracleAuth) DeepCopyInto(out *OracleAuth) { + *out = *in + in.SecretRef.DeepCopyInto(&out.SecretRef) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OracleAuth. +func (in *OracleAuth) DeepCopy() *OracleAuth { + if in == nil { + return nil + } + out := new(OracleAuth) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OracleProvider) DeepCopyInto(out *OracleProvider) { + *out = *in + in.Auth.DeepCopyInto(&out.Auth) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OracleProvider. +func (in *OracleProvider) DeepCopy() *OracleProvider { + if in == nil { + return nil + } + out := new(OracleProvider) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OracleSecretRef) DeepCopyInto(out *OracleSecretRef) { + *out = *in + in.PrivateKey.DeepCopyInto(&out.PrivateKey) + in.Fingerprint.DeepCopyInto(&out.Fingerprint) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OracleSecretRef. +func (in *OracleSecretRef) DeepCopy() *OracleSecretRef { + if in == nil { + return nil + } + out := new(OracleSecretRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecretStore) DeepCopyInto(out *SecretStore) { *out = *in @@ -639,11 +789,31 @@ func (in *SecretStoreProvider) DeepCopyInto(out *SecretStoreProvider) { *out = new(GCPSMProvider) (*in).DeepCopyInto(*out) } + if in.Oracle != nil { + in, out := &in.Oracle, &out.Oracle + *out = new(OracleProvider) + (*in).DeepCopyInto(*out) + } if in.IBM != nil { in, out := &in.IBM, &out.IBM *out = new(IBMProvider) (*in).DeepCopyInto(*out) } + if in.YandexLockbox != nil { + in, out := &in.YandexLockbox, &out.YandexLockbox + *out = new(YandexLockboxProvider) + (*in).DeepCopyInto(*out) + } + if in.Gitlab != nil { + in, out := &in.Gitlab, &out.Gitlab + *out = new(GitlabProvider) + (*in).DeepCopyInto(*out) + } + if in.Alibaba != nil { + in, out := &in.Alibaba, &out.Alibaba + *out = new(AlibabaProvider) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretStoreProvider. @@ -949,3 +1119,35 @@ func (in *VaultProvider) DeepCopy() *VaultProvider { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *YandexLockboxAuth) DeepCopyInto(out *YandexLockboxAuth) { + *out = *in + in.AuthorizedKey.DeepCopyInto(&out.AuthorizedKey) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new YandexLockboxAuth. +func (in *YandexLockboxAuth) DeepCopy() *YandexLockboxAuth { + if in == nil { + return nil + } + out := new(YandexLockboxAuth) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *YandexLockboxProvider) DeepCopyInto(out *YandexLockboxProvider) { + *out = *in + in.Auth.DeepCopyInto(&out.Auth) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new YandexLockboxProvider. +func (in *YandexLockboxProvider) DeepCopy() *YandexLockboxProvider { + if in == nil { + return nil + } + out := new(YandexLockboxProvider) + in.DeepCopyInto(out) + return out +} diff --git a/apis/meta/v1/types.go b/apis/meta/v1/types.go index 0bb3a2514..ffc6c4e5a 100644 --- a/apis/meta/v1/types.go +++ b/apis/meta/v1/types.go @@ -18,7 +18,7 @@ package v1 // In some instances, `key` is a required field. type SecretKeySelector struct { // The name of the Secret resource being referred to. - Name string `json:"name"` + Name string `json:"name,omitempty"` // Namespace of the resource being referred to. Ignored if referent is not cluster-scoped. cluster-scoped defaults // to the namespace of the referent. // +optional diff --git a/deploy/charts/external-secrets/Chart.yaml b/deploy/charts/external-secrets/Chart.yaml index 95090b709..79e74b36b 100644 --- a/deploy/charts/external-secrets/Chart.yaml +++ b/deploy/charts/external-secrets/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: external-secrets description: External secret management for Kubernetes type: application -version: "0.3.3" -appVersion: "v0.3.3" +version: "0.3.5" +appVersion: "v0.3.5" kubeVersion: ">= 1.11.0-0" keywords: - kubernetes-external-secrets diff --git a/deploy/charts/external-secrets/README.md b/deploy/charts/external-secrets/README.md index 5a71de466..7ed99b2fd 100644 --- a/deploy/charts/external-secrets/README.md +++ b/deploy/charts/external-secrets/README.md @@ -4,7 +4,7 @@ [//]: # (README.md generated by gotmpl. DO NOT EDIT.) -![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.3.3](https://img.shields.io/badge/Version-0.3.3-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.3.5](https://img.shields.io/badge/Version-0.3.5-informational?style=flat-square) External secret management for Kubernetes @@ -49,11 +49,13 @@ The command removes all the Kubernetes components associated with the chart and | podAnnotations | object | `{}` | | | podLabels | object | `{}` | | | podSecurityContext | object | `{}` | | +| priorityClassName | string | `""` | Pod priority class name. | | prometheus.enabled | bool | `false` | Specifies whether to expose Service resource for collecting Prometheus metrics | | prometheus.service.port | int | `8080` | | | rbac.create | bool | `true` | Specifies whether role and rolebinding resources should be created. | | replicaCount | int | `1` | | | resources | object | `{}` | | +| scopedNamespace | string | `""` | If set external secrets are only reconciled in the provided namespace | | securityContext | object | `{}` | | | serviceAccount.annotations | object | `{}` | Annotations to add to the service account. | | serviceAccount.create | bool | `true` | Specifies whether a service account should be created. | diff --git a/deploy/charts/external-secrets/templates/deployment.yaml b/deploy/charts/external-secrets/templates/deployment.yaml index 7ef0cbcb9..338b698a8 100644 --- a/deploy/charts/external-secrets/templates/deployment.yaml +++ b/deploy/charts/external-secrets/templates/deployment.yaml @@ -2,6 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: {{ include "external-secrets.fullname" . }} + namespace: {{ .Release.Namespace | quote }} labels: {{- include "external-secrets.labels" . | nindent 4 }} spec: @@ -38,11 +39,14 @@ spec: {{- end }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- if or (.Values.leaderElect) (.Values.extraArgs) }} + {{- if or (.Values.leaderElect) (.Values.scopedNamespace) (.Values.extraArgs) }} args: {{- if .Values.leaderElect }} - --enable-leader-election=true {{- end }} + {{- if .Values.scopedNamespace }} + - --namespace={{ .Values.scopedNamespace }} + {{- end }} {{- range $key, $value := .Values.extraArgs }} {{- if $value }} - --{{ $key }}={{ $value }} @@ -74,3 +78,6 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} diff --git a/deploy/charts/external-secrets/templates/serviceaccount.yaml b/deploy/charts/external-secrets/templates/serviceaccount.yaml index 911638fb4..d3e58f78b 100644 --- a/deploy/charts/external-secrets/templates/serviceaccount.yaml +++ b/deploy/charts/external-secrets/templates/serviceaccount.yaml @@ -3,6 +3,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "external-secrets.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} labels: {{- include "external-secrets.labels" . | nindent 4 }} {{- with .Values.serviceAccount.annotations }} diff --git a/deploy/charts/external-secrets/values.yaml b/deploy/charts/external-secrets/values.yaml index 419b06473..738733fc9 100644 --- a/deploy/charts/external-secrets/values.yaml +++ b/deploy/charts/external-secrets/values.yaml @@ -17,6 +17,10 @@ fullnameOverride: "" # than one instance of external-secrets operates at a time. leaderElect: false +# -- If set external secrets are only reconciled in the +# provided namespace +scopedNamespace: "" + serviceAccount: # -- Specifies whether a service account should be created. create: true @@ -66,3 +70,6 @@ nodeSelector: {} tolerations: [] affinity: {} + +# -- Pod priority class name. +priorityClassName: "" diff --git a/deploy/crds/external-secrets.io_clustersecretstores.yaml b/deploy/crds/external-secrets.io_clustersecretstores.yaml index eac6a56fc..f1000da0e 100644 --- a/deploy/crds/external-secrets.io_clustersecretstores.yaml +++ b/deploy/crds/external-secrets.io_clustersecretstores.yaml @@ -54,6 +54,73 @@ spec: maxProperties: 1 minProperties: 1 properties: + alibaba: + description: Alibaba configures this store to sync secrets using + Alibaba Cloud provider + properties: + auth: + description: AlibabaAuth contains a secretRef for credentials. + properties: + secretRef: + description: AlibabaAuthSecretRef holds secret references + for Alibaba credentials. + properties: + accessKeyIDSecretRef: + description: The AccessKeyID is used for authentication + properties: + key: + description: The key of the entry in the Secret + resource's `data` field to be used. Some instances + of this field may be defaulted, in others it + may be required. + type: string + name: + description: The name of the Secret resource being + referred to. + type: string + namespace: + description: Namespace of the resource being referred + to. Ignored if referent is not cluster-scoped. + cluster-scoped defaults to the namespace of + the referent. + type: string + type: object + accessKeySecretSecretRef: + description: The AccessKeySecret is used for authentication + properties: + key: + description: The key of the entry in the Secret + resource's `data` field to be used. Some instances + of this field may be defaulted, in others it + may be required. + type: string + name: + description: The name of the Secret resource being + referred to. + type: string + namespace: + description: Namespace of the resource being referred + to. Ignored if referent is not cluster-scoped. + cluster-scoped defaults to the namespace of + the referent. + type: string + type: object + required: + - accessKeyIDSecretRef + - accessKeySecretSecretRef + type: object + required: + - secretRef + type: object + endpoint: + type: string + regionID: + description: Alibaba Region to be used for the provider + type: string + required: + - auth + - regionID + type: object aws: description: AWS configures this store to sync secrets using AWS Secret Manager provider @@ -108,8 +175,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object secretAccessKeySecretRef: description: The SecretAccessKey is used for authentication @@ -130,8 +195,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object type: object type: object @@ -179,8 +242,6 @@ spec: to. Ignored if referent is not cluster-scoped. cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object clientSecret: description: The Azure ClientSecret of the service principle @@ -200,8 +261,6 @@ spec: to. Ignored if referent is not cluster-scoped. cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object required: - clientId @@ -249,8 +308,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object type: object required: @@ -259,6 +316,49 @@ spec: projectID: description: ProjectID project where secret is located type: string + type: object + gitlab: + description: GItlab configures this store to sync secrets using + Gitlab Variables provider + properties: + auth: + description: Auth configures how secret-manager authenticates + with a GitLab instance. + properties: + SecretRef: + properties: + accessToken: + description: AccessToken is used for authentication. + properties: + key: + description: The key of the entry in the Secret + resource's `data` field to be used. Some instances + of this field may be defaulted, in others it + may be required. + type: string + name: + description: The name of the Secret resource being + referred to. + type: string + namespace: + description: Namespace of the resource being referred + to. Ignored if referent is not cluster-scoped. + cluster-scoped defaults to the namespace of + the referent. + type: string + type: object + type: object + required: + - SecretRef + type: object + projectID: + description: ProjectID specifies a project where secrets are + located. + type: string + url: + description: URL configures the GitLab instance URL. Defaults + to https://gitlab.com/. + type: string required: - auth type: object @@ -291,8 +391,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object type: object required: @@ -305,6 +403,76 @@ spec: required: - auth type: object + oracle: + description: Oracle configures this store to sync secrets using + Oracle Vault provider + properties: + auth: + description: Auth configures how secret-manager authenticates + with the Oracle Vault. + properties: + secretRef: + description: SecretRef to pass through sensitive information. + properties: + fingerprint: + description: projectID is an access token specific + to the secret. + properties: + key: + description: The key of the entry in the Secret + resource's `data` field to be used. Some instances + of this field may be defaulted, in others it + may be required. + type: string + name: + description: The name of the Secret resource being + referred to. + type: string + namespace: + description: Namespace of the resource being referred + to. Ignored if referent is not cluster-scoped. + cluster-scoped defaults to the namespace of + the referent. + type: string + type: object + privatekey: + description: The Access Token is used for authentication + properties: + key: + description: The key of the entry in the Secret + resource's `data` field to be used. Some instances + of this field may be defaulted, in others it + may be required. + type: string + name: + description: The name of the Secret resource being + referred to. + type: string + namespace: + description: Namespace of the resource being referred + to. Ignored if referent is not cluster-scoped. + cluster-scoped defaults to the namespace of + the referent. + type: string + type: object + type: object + required: + - secretRef + type: object + region: + description: projectID is an access token specific to the + secret. + type: string + tenancy: + description: projectID is an access token specific to the + secret. + type: string + user: + description: User is an access OCID specific to the account. + type: string + required: + - auth + type: object vault: description: Vault configures this store to sync secrets using Hashi provider @@ -351,8 +519,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object required: - path @@ -384,8 +550,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object secretRef: description: SecretRef to a key in a Secret resource @@ -408,8 +572,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object type: object jwt: @@ -441,8 +603,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object type: object kubernetes: @@ -483,8 +643,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object serviceAccountRef: description: Optional service account field containing @@ -537,8 +695,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object username: description: Username is a LDAP user name used to @@ -566,8 +722,6 @@ spec: to. Ignored if referent is not cluster-scoped. cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object type: object caBundle: @@ -608,6 +762,39 @@ spec: - path - server type: object + yandexlockbox: + description: YandexLockbox configures this store to sync secrets + using Yandex Lockbox provider + properties: + apiEndpoint: + description: Yandex.Cloud API endpoint (e.g. 'api.cloud.yandex.net:443') + type: string + auth: + description: Auth defines the information necessary to authenticate + against Yandex Lockbox + properties: + authorizedKeySecretRef: + description: The authorized key used for authentication + properties: + key: + description: The key of the entry in the Secret resource's + `data` field to be used. Some instances of this + field may be defaulted, in others it may be required. + type: string + name: + description: The name of the Secret resource being + referred to. + type: string + namespace: + description: Namespace of the resource being referred + to. Ignored if referent is not cluster-scoped. cluster-scoped + defaults to the namespace of the referent. + type: string + type: object + type: object + required: + - auth + type: object type: object required: - provider diff --git a/deploy/crds/external-secrets.io_secretstores.yaml b/deploy/crds/external-secrets.io_secretstores.yaml index b84336b5a..2b11581e2 100644 --- a/deploy/crds/external-secrets.io_secretstores.yaml +++ b/deploy/crds/external-secrets.io_secretstores.yaml @@ -54,6 +54,73 @@ spec: maxProperties: 1 minProperties: 1 properties: + alibaba: + description: Alibaba configures this store to sync secrets using + Alibaba Cloud provider + properties: + auth: + description: AlibabaAuth contains a secretRef for credentials. + properties: + secretRef: + description: AlibabaAuthSecretRef holds secret references + for Alibaba credentials. + properties: + accessKeyIDSecretRef: + description: The AccessKeyID is used for authentication + properties: + key: + description: The key of the entry in the Secret + resource's `data` field to be used. Some instances + of this field may be defaulted, in others it + may be required. + type: string + name: + description: The name of the Secret resource being + referred to. + type: string + namespace: + description: Namespace of the resource being referred + to. Ignored if referent is not cluster-scoped. + cluster-scoped defaults to the namespace of + the referent. + type: string + type: object + accessKeySecretSecretRef: + description: The AccessKeySecret is used for authentication + properties: + key: + description: The key of the entry in the Secret + resource's `data` field to be used. Some instances + of this field may be defaulted, in others it + may be required. + type: string + name: + description: The name of the Secret resource being + referred to. + type: string + namespace: + description: Namespace of the resource being referred + to. Ignored if referent is not cluster-scoped. + cluster-scoped defaults to the namespace of + the referent. + type: string + type: object + required: + - accessKeyIDSecretRef + - accessKeySecretSecretRef + type: object + required: + - secretRef + type: object + endpoint: + type: string + regionID: + description: Alibaba Region to be used for the provider + type: string + required: + - auth + - regionID + type: object aws: description: AWS configures this store to sync secrets using AWS Secret Manager provider @@ -108,8 +175,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object secretAccessKeySecretRef: description: The SecretAccessKey is used for authentication @@ -130,8 +195,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object type: object type: object @@ -179,8 +242,6 @@ spec: to. Ignored if referent is not cluster-scoped. cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object clientSecret: description: The Azure ClientSecret of the service principle @@ -200,8 +261,6 @@ spec: to. Ignored if referent is not cluster-scoped. cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object required: - clientId @@ -249,8 +308,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object type: object required: @@ -259,6 +316,49 @@ spec: projectID: description: ProjectID project where secret is located type: string + type: object + gitlab: + description: GItlab configures this store to sync secrets using + Gitlab Variables provider + properties: + auth: + description: Auth configures how secret-manager authenticates + with a GitLab instance. + properties: + SecretRef: + properties: + accessToken: + description: AccessToken is used for authentication. + properties: + key: + description: The key of the entry in the Secret + resource's `data` field to be used. Some instances + of this field may be defaulted, in others it + may be required. + type: string + name: + description: The name of the Secret resource being + referred to. + type: string + namespace: + description: Namespace of the resource being referred + to. Ignored if referent is not cluster-scoped. + cluster-scoped defaults to the namespace of + the referent. + type: string + type: object + type: object + required: + - SecretRef + type: object + projectID: + description: ProjectID specifies a project where secrets are + located. + type: string + url: + description: URL configures the GitLab instance URL. Defaults + to https://gitlab.com/. + type: string required: - auth type: object @@ -291,8 +391,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object type: object required: @@ -305,6 +403,76 @@ spec: required: - auth type: object + oracle: + description: Oracle configures this store to sync secrets using + Oracle Vault provider + properties: + auth: + description: Auth configures how secret-manager authenticates + with the Oracle Vault. + properties: + secretRef: + description: SecretRef to pass through sensitive information. + properties: + fingerprint: + description: projectID is an access token specific + to the secret. + properties: + key: + description: The key of the entry in the Secret + resource's `data` field to be used. Some instances + of this field may be defaulted, in others it + may be required. + type: string + name: + description: The name of the Secret resource being + referred to. + type: string + namespace: + description: Namespace of the resource being referred + to. Ignored if referent is not cluster-scoped. + cluster-scoped defaults to the namespace of + the referent. + type: string + type: object + privatekey: + description: The Access Token is used for authentication + properties: + key: + description: The key of the entry in the Secret + resource's `data` field to be used. Some instances + of this field may be defaulted, in others it + may be required. + type: string + name: + description: The name of the Secret resource being + referred to. + type: string + namespace: + description: Namespace of the resource being referred + to. Ignored if referent is not cluster-scoped. + cluster-scoped defaults to the namespace of + the referent. + type: string + type: object + type: object + required: + - secretRef + type: object + region: + description: projectID is an access token specific to the + secret. + type: string + tenancy: + description: projectID is an access token specific to the + secret. + type: string + user: + description: User is an access OCID specific to the account. + type: string + required: + - auth + type: object vault: description: Vault configures this store to sync secrets using Hashi provider @@ -351,8 +519,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object required: - path @@ -384,8 +550,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object secretRef: description: SecretRef to a key in a Secret resource @@ -408,8 +572,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object type: object jwt: @@ -441,8 +603,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object type: object kubernetes: @@ -483,8 +643,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object serviceAccountRef: description: Optional service account field containing @@ -537,8 +695,6 @@ spec: cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object username: description: Username is a LDAP user name used to @@ -566,8 +722,6 @@ spec: to. Ignored if referent is not cluster-scoped. cluster-scoped defaults to the namespace of the referent. type: string - required: - - name type: object type: object caBundle: @@ -608,6 +762,39 @@ spec: - path - server type: object + yandexlockbox: + description: YandexLockbox configures this store to sync secrets + using Yandex Lockbox provider + properties: + apiEndpoint: + description: Yandex.Cloud API endpoint (e.g. 'api.cloud.yandex.net:443') + type: string + auth: + description: Auth defines the information necessary to authenticate + against Yandex Lockbox + properties: + authorizedKeySecretRef: + description: The authorized key used for authentication + properties: + key: + description: The key of the entry in the Secret resource's + `data` field to be used. Some instances of this + field may be defaulted, in others it may be required. + type: string + name: + description: The name of the Secret resource being + referred to. + type: string + namespace: + description: Namespace of the resource being referred + to. Ignored if referent is not cluster-scoped. cluster-scoped + defaults to the namespace of the referent. + type: string + type: object + type: object + required: + - auth + type: object type: object required: - provider diff --git a/docs/contributing-devguide.md b/docs/contributing-devguide.md index eb4254092..2088a0413 100644 --- a/docs/contributing-devguide.md +++ b/docs/contributing-devguide.md @@ -8,6 +8,17 @@ git clone https://github.com/external-secrets/external-secrets.git cd external-secrets ``` +If you want to run controller tests you also need to install kubebuilder's `envtest`: + +``` +export KUBEBUILDER_TOOLS_VERSION='1.20.2' # check for latest version or a version that has support to what you are testing + +curl -sSLo envtest-bins.tar.gz "https://storage.googleapis.com/kubebuilder-tools/kubebuilder-tools-$KUBEBUILDER_TOOLS_VERSION-linux-amd64.tar.gz" + +sudo mkdir -p /usr/local/kubebuilder +sudo tar -C /usr/local/kubebuilder --strip-components=1 -zvxf envtest-bins.tar.gz +``` + ## Building & Testing The project uses the `make` build system. It'll run code generators, tests and @@ -33,21 +44,19 @@ make docs ## Installing -To install the External Secret Operator's CRDs into a Kubernetes Cluster run: +To install the External Secret Operator into a Kubernetes Cluster run: + +```shell +helm repo add external-secrets https://charts.external-secrets.io +helm repo update +helm install external-secrets external-secrets/external-secrets +``` + +You can alternatively run the controller on your host system for development purposes: + ```shell make crds.install -``` - -Apply the sample resources: -```shell -kubectl apply -f docs/snippets/basic-secret-store.yaml -kubectl apply -f docs/snippets/basic-external-secret.yaml -``` - -You can run the controller on your host system for development purposes: - -```shell make run ``` @@ -57,6 +66,20 @@ To remove the CRDs run: make crds.uninstall ``` +If you need to test some other k8s integrations and need the operator to be deployed to the actuall cluster while developing, you can use the following workflow: + +``` +kind create cluster --name external-secrets + +export TAG=v2 +export IMAGE=eso-local + +docker build . -t $IMAGE:$TAG --build-arg TARGETARCH=amd64 --build-arg TARGETOS=linux + +make helm.generate +helm upgrade --install external-secrets ./deploy/charts/external-secrets/ --set image.repository=$IMAGE --set image.tag=$TAG +``` + !!! note "Contributing Flow" The HOW TO guide for contributing is at the [Contributing Process](contributing-process.md) page. diff --git a/docs/pictures/screenshot_API_key.png b/docs/pictures/screenshot_API_key.png new file mode 100644 index 000000000..6cadb57d0 Binary files /dev/null and b/docs/pictures/screenshot_API_key.png differ diff --git a/docs/pictures/screenshot_fingerprint.png b/docs/pictures/screenshot_fingerprint.png new file mode 100644 index 000000000..ea5eaa8e1 Binary files /dev/null and b/docs/pictures/screenshot_fingerprint.png differ diff --git a/docs/pictures/screenshot_gitlab_projectID.png b/docs/pictures/screenshot_gitlab_projectID.png new file mode 100644 index 000000000..6edb9e602 Binary files /dev/null and b/docs/pictures/screenshot_gitlab_projectID.png differ diff --git a/docs/pictures/screenshot_gitlab_token.png b/docs/pictures/screenshot_gitlab_token.png new file mode 100644 index 000000000..d526345d0 Binary files /dev/null and b/docs/pictures/screenshot_gitlab_token.png differ diff --git a/docs/pictures/screenshot_gitlab_token_created.png b/docs/pictures/screenshot_gitlab_token_created.png new file mode 100644 index 000000000..7f05861e7 Binary files /dev/null and b/docs/pictures/screenshot_gitlab_token_created.png differ diff --git a/docs/pictures/screenshot_region.png b/docs/pictures/screenshot_region.png new file mode 100644 index 000000000..1886acdbc Binary files /dev/null and b/docs/pictures/screenshot_region.png differ diff --git a/docs/pictures/screenshot_tenancy_OCID.png b/docs/pictures/screenshot_tenancy_OCID.png new file mode 100644 index 000000000..fb8c76525 Binary files /dev/null and b/docs/pictures/screenshot_tenancy_OCID.png differ diff --git a/docs/pictures/screenshot_user_OCID.png b/docs/pictures/screenshot_user_OCID.png new file mode 100644 index 000000000..178888d8f Binary files /dev/null and b/docs/pictures/screenshot_user_OCID.png differ diff --git a/docs/provider-aws-parameter-store.md b/docs/provider-aws-parameter-store.md index 9beba6d26..0eefe34c0 100644 --- a/docs/provider-aws-parameter-store.md +++ b/docs/provider-aws-parameter-store.md @@ -19,7 +19,7 @@ way users of the `SecretStore` can only access the secrets necessary. ### IAM Policy -Create a IAM Policy to pin down access to secrets matching `dev-*`, for futher information see [AWS Documentation](https://docs.aws.amazon.com/systems-manager/latest/userguide/sysman-paramstore-access.html): +Create a IAM Policy to pin down access to secrets matching `dev-*`, for further information see [AWS Documentation](https://docs.aws.amazon.com/systems-manager/latest/userguide/sysman-paramstore-access.html): ``` json { diff --git a/docs/provider-gitlab-project-variables.md b/docs/provider-gitlab-project-variables.md new file mode 100644 index 000000000..1e3bcde69 --- /dev/null +++ b/docs/provider-gitlab-project-variables.md @@ -0,0 +1,54 @@ +## Gitlab Project Variables + +External Secrets Operator integrates with [Gitlab API](https://docs.gitlab.com/ee/api/project_level_variables.html) to sync Gitlab project variables to secrets held on the Kubernetes cluster. + +### Authentication + +The API requires an access token and project ID. To create a new access token, go to your user settings and select 'access tokens'. Give your token a name, expiration date, and select the permissions required (Note 'api' is required). + +![token-details](./pictures/screenshot_gitlab_token.png) + +Click 'Create personal access token', and your token will be generated and displayed on screen. Copy or save this token since you can't access it again. +![token-created](./pictures/screenshot_gitlab_token_created.png) + + + +### Access Token secret + +Create a secret containing your access token: + +```yaml +{% include 'gitlab-credentials-secret.yaml' %} +``` + +### Update secret store +Be sure the `gitlab` provider is listed in the `Kind=SecretStore` and the ProjectID is set. If you are not using `https://gitlab.com`, you must set the `url` field as well. + +```yaml +{% include 'gitlab-secret-store.yaml' %} +``` + +Your project ID can be found on your project's page. +![projectID](./pictures/screenshot_gitlab_projectID.png) + +### Creating external secret + +To sync a Gitlab variable to a secret on the Kubernetes cluster, a `Kind=ExternalSecret` is needed. + +```yaml +{% include 'gitlab-external-secret.yaml' %} +``` + +#### Using DataFrom + +DataFrom can be used to get a variable as a JSON string and attempt to parse it. + +```yaml +{% include 'gitlab-external-secret-json.yaml' %} +``` + +### Getting the Kubernetes secret +The operator will fetch the project variable and inject it as a `Kind=Secret`. +``` +kubectl get secret gitlab-secret-to-create -o jsonpath='{.data.secretKey}' | base64 -d +``` diff --git a/docs/provider-google-secrets-manager.md b/docs/provider-google-secrets-manager.md index b3f622a70..8f43e5a4b 100644 --- a/docs/provider-google-secrets-manager.md +++ b/docs/provider-google-secrets-manager.md @@ -2,11 +2,7 @@ External Secrets Operator integrates with [GCP Secret Manager](https://cloud.google.com/secret-manager) for secret management. -### Authentication - -At the moment, we only support [service account key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) authentication. - -#### Service account key authentication +### Service account key authentication A service account key is created and the JSON keyfile is stored in a `Kind=Secret`. The `project_id` and `private_key` should be configured for the project. @@ -33,3 +29,63 @@ The operator will fetch the GCP Secret Manager secret and inject it as a `Kind=S ``` kubectl get secret secret-to-be-created -n | -o jsonpath='{.data.dev-secret-test}' | base64 -d ``` + +## Authentication with Workload Identity + +This makes it possible for your Google Kubernetes Engine (GKE) applications to consume services provided by Google APIs, namely Secrets Manager service in this case. + +Here we will assume that you installed ESO using helm and that you named the chart installation `external-secrets` and the namespace where it lives `es` like: + +```sh +helm install external-secrets external-secrets/external-secrets --namespace es +``` + +Then most of the resources would have this name, the important one here being the k8s service account attached to the external-secrets operator deployment: + +``` +# ... + containers: + - image: ghcr.io/external-secrets/external-secrets:vVERSION + name: external-secrets + ports: + - containerPort: 8080 + protocol: TCP + restartPolicy: Always + schedulerName: default-scheduler + serviceAccount: external-secrets + serviceAccountName: external-secrets # <--- here +``` + +### Following the documentation + +You can find the documentation for Workload Identity under [this url](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity). We will walk you through how to navigate it here. + +#### Changing Values + +Search [the documment](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) for this editable values and change them to your values: + +- CLUSTER_NAME: The name of your cluster +- PROJECT_ID: Your project ID (not your Project number nor your Project name) +- K8S_NAMESPACE: For us folowing these steps here it will be `es`, but this will be the namespace where you deployed the external-secrets operator +- KSA_NAME: external-secrets (if you are not creating a new one to attach to the deployemnt) +- GSA_NAME: external-secrets for simplicity, or something else if you have to follow different naming convetions for cloud resources +- ROLE_NAME: roles/secretmanager.secretAccessor so you make the pod only be able to access secrets on Secret Manager + +#### Following through + +You can follow through the documentation and adapt it to your specific use case. If you want to just use the serviceaccount that we deployed with the helm chart, for example, you don't need to create a new service account on 2 of [Authenticating to Google Cloud](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#authenticating_to). + +#### SecretStore with WorkloadIdentity + +To use workload identity you can just omit the auth field of the secret store and let the operator client fall back to defaults using the roles attached to your service account. + +``` +apiVersion: external-secrets.io/v1alpha1 +kind: SecretStore +metadata: + name: example +spec: + provider: + gcpsm: + projectID: pid +``` \ No newline at end of file diff --git a/docs/provider-oracle-vault.md b/docs/provider-oracle-vault.md new file mode 100644 index 000000000..baf8c7690 --- /dev/null +++ b/docs/provider-oracle-vault.md @@ -0,0 +1,54 @@ +## Oracle Vault + +External Secrets Operator integrates with [OCI API](https://github.com/oracle/oci-go-sdk) to sync secret on the Oracle Vault to secrets held on the Kubernetes cluster. + +### Authentication + +The API requires a userOCID, tenancyOCID, fingerprint, key file and a region. The fingerprint and key file should be supplied in the secret with the rest being provided in the secret store. + +See url for what region you you are accessing. +![userOCID-details](./pictures/screenshot_region.png) + +Select tenancy in the top right to see your user OCID as shown below. +![tenancyOCID-details](./pictures/tenancy.png) + +Select your user in the top right to see your user OCID as shown below. +![region-details](./pictures/screenshot_user_OCID.png) + + +#### Service account key authentication + +Create a secret containing your private key and fingerprint: + +```yaml +{% include 'oracle-credentials-secret.yaml' %} +``` + +Your fingerprint will be attatched to your API key, once it has been generated. Found on the same page as the user OCID. +![fingerprint-details](./pictures/screenshot_fingerprint.png) + +Once you click "Add API Key" you will be shown the following, where you can download the RSA key in the necessary PEM format for API requests. +This will automatically generate a fingerprint. +![API-key-details](./pictures/screenshot_API_key.png) + +### Update secret store +Be sure the `oracle` provider is listed in the `Kind=SecretStore` + +```yaml +{% include 'oracle-secret-store.yaml' %} +``` + +### Creating external secret + +To create a kubernetes secret from the Oracle Cloud Interface secret a`Kind=ExternalSecret` is needed. + +```yaml +{% include 'oracle-external-secret.yaml' %} +``` + + +### Getting the Kubernetes secret +The operator will fetch the project variable and inject it as a `Kind=Secret`. +``` +kubectl get secret oracle-secret-to-create -o jsonpath='{.data.dev-secret-test}' | base64 -d +``` \ No newline at end of file diff --git a/docs/provider-yandex-lockbox.md b/docs/provider-yandex-lockbox.md new file mode 100644 index 000000000..8eb5cb4c9 --- /dev/null +++ b/docs/provider-yandex-lockbox.md @@ -0,0 +1,86 @@ +## Yandex Lockbox + +External Secrets Operator integrates with [Yandex Lockbox](https://cloud.yandex.com/docs/lockbox/) +for secret management. + +### Prerequisites +* [External Secrets Operator installed](../guides-getting-started/#installing-with-helm) +* [Yandex.Cloud CLI installed](https://cloud.yandex.com/docs/cli/quickstart) + +### Authentication +At the moment, [authorized key](https://cloud.yandex.com/docs/iam/concepts/authorization/key) authentication is only supported: + +* Create a [service account](https://cloud.yandex.com/docs/iam/concepts/users/service-accounts) in Yandex.Cloud: +```bash +yc iam service-account create --name eso-service-account +``` +* Create an authorized key for the service account and save it to `authorized-key.json` file: +```bash +yc iam key create \ + --service-account-name eso-service-account \ + --output authorized-key.json +``` +* Create a k8s secret containing the authorized key saved above: +```bash +kubectl create secret generic yc-auth --from-file=authorized-key=authorized-key.json +``` +* Create a [SecretStore](../api-secretstore/) pointing to `yc-auth` k8s secret: +```yaml +apiVersion: external-secrets.io/v1alpha1 +kind: SecretStore +metadata: + name: secret-store +spec: + provider: + yandexlockbox: + auth: + authorizedKeySecretRef: + name: yc-auth + key: authorized-key +``` + +### Creating external secret +To make External Secrets Operator sync a k8s secret with a Lockbox secret: + +* Create a Lockbox secret, if not already created: +```bash +yc lockbox secret create \ + --name lockbox-secret \ + --payload '[{"key": "password","textValue": "p@$$w0rd"}]' +``` +* Assign the [`lockbox.payloadViewer`](https://cloud.yandex.com/docs/lockbox/security/#roles-list) role + for accessing the `lockbox-secret` payload to the service account used for authentication: +```bash +yc lockbox secret add-access-binding \ + --name lockbox-secret \ + --service-account-name eso-service-account \ + --role lockbox.payloadViewer +``` +Run the following command to ensure that the correct access binding has been added: +```bash +yc lockbox secret list-access-bindings --name lockbox-secret +``` +* Create an [ExternalSecret](../api-externalsecret/) pointing to `secret-store` and `lockbox-secret`: +```yaml +apiVersion: external-secrets.io/v1alpha1 +kind: ExternalSecret +metadata: + name: external-secret +spec: + refreshInterval: 1h + secretStoreRef: + name: secret-store + kind: SecretStore + target: + name: k8s-secret # the target k8s secret name + data: + - secretKey: password # the target k8s secret key + remoteRef: + key: ***** # ID of lockbox-secret + property: password # (optional) payload entry key of lockbox-secret +``` + +The operator will fetch the Yandex Lockbox secret and inject it as a `Kind=Secret` +```yaml +kubectl get secret k8s-secret -n | -o jsonpath='{.data.password}' | base64 -d +``` diff --git a/docs/snippets/gitlab-credentials-secret.yaml b/docs/snippets/gitlab-credentials-secret.yaml new file mode 100644 index 000000000..cca2557f3 --- /dev/null +++ b/docs/snippets/gitlab-credentials-secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: gitlab-secret + labels: + type: gitlab +type: Opaque +stringData: + token: "**access token goes here**" diff --git a/docs/snippets/gitlab-external-secret-json.yaml b/docs/snippets/gitlab-external-secret-json.yaml new file mode 100644 index 000000000..9733ab805 --- /dev/null +++ b/docs/snippets/gitlab-external-secret-json.yaml @@ -0,0 +1,18 @@ +apiVersion: external-secrets.io/v1alpha1 +kind: ExternalSecret +metadata: + name: gitlab-external-secret-example +spec: + refreshInterval: 1h + + secretStoreRef: + kind: SecretStore + name: gitlab-secret-store # Must match SecretStore on the cluster + + target: + name: gitlab-secret-to-create # Name for the secret to be created on the cluster + creationPolicy: Owner + + # each secret name in the KV will be used as the secret key in the SECRET k8s target object + dataFrom: + - key: "myJsonVariable" # Key of the variable on Gitlab \ No newline at end of file diff --git a/docs/snippets/gitlab-external-secret.yaml b/docs/snippets/gitlab-external-secret.yaml new file mode 100644 index 000000000..c48228f68 --- /dev/null +++ b/docs/snippets/gitlab-external-secret.yaml @@ -0,0 +1,19 @@ +apiVersion: external-secrets.io/v1alpha1 +kind: ExternalSecret +metadata: + name: gitlab-external-secret-example +spec: + refreshInterval: 1h + + secretStoreRef: + kind: SecretStore + name: gitlab-secret-store # Must match SecretStore on the cluster + + target: + name: gitlab-secret-to-create # Name for the secret to be created on the cluster + creationPolicy: Owner + + data: + - secretKey: secretKey # Key given to the secret to be created on the cluster + remoteRef: + key: myGitlabVariable # Key of the variable on Gitlab \ No newline at end of file diff --git a/docs/snippets/gitlab-secret-store.yaml b/docs/snippets/gitlab-secret-store.yaml new file mode 100644 index 000000000..d2ae32793 --- /dev/null +++ b/docs/snippets/gitlab-secret-store.yaml @@ -0,0 +1,15 @@ +apiVersion: external-secrets.io/v1alpha1 +kind: SecretStore +metadata: + name: gitlab-secret-store +spec: + provider: + # provider type: gitlab + gitlab: + # url: https://gitlab.mydomain.com/ + auth: + SecretRef: + accessToken: + name: gitlab-secret + key: token + projectID: "**project ID goes here**" \ No newline at end of file diff --git a/docs/snippets/oracle-credentials-secret.yaml b/docs/snippets/oracle-credentials-secret.yaml new file mode 100644 index 000000000..108e761f1 --- /dev/null +++ b/docs/snippets/oracle-credentials-secret.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: oracle-secret + labels: + type: oracle +type: Opaque +stringData: + privateKey: + fingerprint: \ No newline at end of file diff --git a/docs/snippets/oracle-external-secret.yaml b/docs/snippets/oracle-external-secret.yaml new file mode 100644 index 000000000..e47a557ff --- /dev/null +++ b/docs/snippets/oracle-external-secret.yaml @@ -0,0 +1,16 @@ +apiVersion: external-secrets.io/v1alpha1 +kind: ExternalSecret +metadata: + name: example +spec: + refreshInterval: 0.03m + secretStoreRef: + kind: SecretStore + name: example # Must match SecretStore on the cluster + target: + name: secret-to-be-created # Name for the secret on the cluster + creationPolicy: Owner + data: + - secretKey: + remoteRef: + key: \ No newline at end of file diff --git a/docs/snippets/oracle-secret-store.yaml b/docs/snippets/oracle-secret-store.yaml new file mode 100644 index 000000000..46708221d --- /dev/null +++ b/docs/snippets/oracle-secret-store.yaml @@ -0,0 +1,18 @@ +apiVersion: external-secrets.io/v1alpha1 +kind: SecretStore +metadata: + name: example +spec: + provider: + oracle: #Needs to match value in secretstore_types.go + user: + tenancy: + region: + auth: + secretRef: + privatekey: + name: oracle-secret + key: privateKey #Needs to match stringData val in secret_oracle.yml + fingerprint: + name: oracle-secret + key: fingerprint \ No newline at end of file diff --git a/docs/snippets/provider-aws-access.md b/docs/snippets/provider-aws-access.md index c11558cd3..af470eb28 100644 --- a/docs/snippets/provider-aws-access.md +++ b/docs/snippets/provider-aws-access.md @@ -19,6 +19,7 @@ spec: provider: aws: service: SecretsManager + region: eu-central-1 # optional: do a sts:assumeRole before fetching secrets role: team-b ``` @@ -37,6 +38,7 @@ spec: provider: aws: service: SecretsManager + region: eu-central-1 # optional: assume role before fetching secrets role: team-b auth: @@ -78,6 +80,7 @@ spec: provider: aws: service: SecretsManager + region: eu-central-1 auth: jwt: serviceAccountRef: diff --git a/docs/snippets/vault-approle-store.yaml b/docs/snippets/vault-approle-store.yaml index e786b6742..306bc8874 100644 --- a/docs/snippets/vault-approle-store.yaml +++ b/docs/snippets/vault-approle-store.yaml @@ -18,7 +18,8 @@ spec: path: "approle" # RoleID configured in the App Role authentication backend roleId: "db02de05-fa39-4855-059b-67221c5c2f63" + # Reference to a key in a K8 Secret that contains the App Role SecretId secretRef: name: "my-secret" namespace: "secret-admin" - key: "vault-token" + key: "secret-id" diff --git a/docs/spec.md b/docs/spec.md index c2bc64649..70b89fa0c 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -543,7 +543,9 @@ ExternalSecretStatus Description -

"Ready"

+

"Deleted"

+ +

"Ready"

@@ -1151,6 +1153,7 @@ GCPSMAuth +(Optional)

Auth defines the information necessary to authenticate against GCP

@@ -1173,6 +1176,119 @@ string

GenericStore is a common interface for interacting with ClusterSecretStore or a namespaced SecretStore.

+

GitlabAuth +

+

+(Appears on: +GitlabProvider) +

+

+

+ + + + + + + + + + + + + +
FieldDescription
+SecretRef
+ + +GitlabSecretRef + + +		 +
+

GitlabProvider +

+

+(Appears on: +SecretStoreProvider) +

+

+

Configures a store to sync secrets with a GitLab instance.

+

+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+url
+ +string + +		 +

URL configures the GitLab instance URL. Defaults to https://gitlab.com/.

+
+auth
+ + +GitlabAuth + + +		 +

Auth configures how secret-manager authenticates with a GitLab instance.

+
+projectID
+ +string + +		 +

ProjectID specifies a project where secrets are located.

+
+

GitlabSecretRef +

+

+(Appears on: +GitlabAuth) +

+

+

+ + + + + + + + + + + + + +
FieldDescription
+accessToken
+ +github.com/external-secrets/external-secrets/apis/meta/v1.SecretKeySelector + +		 +

AccessToken is used for authentication.

+

IBMAuth

@@ -1466,6 +1582,34 @@ IBMProvider

IBM configures this store to sync secrets using IBM Cloud provider

+ + +yandexlockbox
+ + +YandexLockboxProvider + + + + +(Optional) +

YandexLockbox configures this store to sync secrets using Yandex Lockbox provider

+ + + + +gitlab
+ + +GitlabProvider + + + + +(Optional) +

GItlab configures this store to sync secrets using Gitlab Variables provider

+ +

SecretStoreRef @@ -2274,6 +2418,80 @@ are used to validate the TLS connection.

+

YandexLockboxAuth +

+

+(Appears on: +YandexLockboxProvider) +

+

+

+ + + + + + + + + + + + + +
FieldDescription
+authorizedKeySecretRef
+ +github.com/external-secrets/external-secrets/apis/meta/v1.SecretKeySelector + +		 +(Optional) +

The authorized key used for authentication

+
+

YandexLockboxProvider +

+

+(Appears on: +SecretStoreProvider) +

+

+

YandexLockboxProvider Configures a store to sync secrets using the Yandex Lockbox provider.

+

+ + + + + + + + + + + + + + + + + +
FieldDescription
+apiEndpoint
+ +string + +		 +(Optional) +

Yandex.Cloud API endpoint (e.g. ‘api.cloud.yandex.net:443’)

+
+auth
+ + +YandexLockboxAuth + + +		 +

Auth defines the information necessary to authenticate against Yandex Lockbox

+

Generated with gen-crd-api-reference-docs. diff --git a/e2e/.DS_Store b/e2e/.DS_Store new file mode 100644 index 000000000..2ad02e817 Binary files /dev/null and b/e2e/.DS_Store differ diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index aefcdab0e..d74793ffe 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -40,6 +40,9 @@ var _ = SynchronizedBeforeSuite(func() []byte { By("installing eso") addon.InstallGlobalAddon(addon.NewESO(), cfg) + + By("installing scoped eso") + addon.InstallGlobalAddon(addon.NewScopedESO(), cfg) return nil }, func([]byte) {}) diff --git a/e2e/framework/addon/eso.go b/e2e/framework/addon/eso.go index c09fa3875..428e55c7b 100644 --- a/e2e/framework/addon/eso.go +++ b/e2e/framework/addon/eso.go @@ -27,3 +27,14 @@ func NewESO() *ESO { }, } } + +func NewScopedESO() *ESO { + return &ESO{ + &HelmChart{ + Namespace: "default", + ReleaseName: "eso-aws-sm", + Chart: "/k8s/deploy/charts/external-secrets", + Values: []string{"/k8s/eso.scoped.values.yaml"}, + }, + } +} diff --git a/e2e/framework/eso.go b/e2e/framework/eso.go index b17d4895c..ab9cb8256 100644 --- a/e2e/framework/eso.go +++ b/e2e/framework/eso.go @@ -11,6 +11,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package framework import ( @@ -23,6 +24,8 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + + esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" ) // WaitForSecretValue waits until a secret comes into existence and compares the secret.Data @@ -52,6 +55,12 @@ func equalSecrets(exp, ts *v1.Secret) bool { return false } + // secret contains data hash property which must be ignored + delete(ts.ObjectMeta.Annotations, esv1alpha1.AnnotationDataHash) + if len(ts.ObjectMeta.Annotations) == 0 { + ts.ObjectMeta.Annotations = nil + } + expAnnotations, _ := json.Marshal(exp.ObjectMeta.Annotations) tsAnnotations, _ := json.Marshal(ts.ObjectMeta.Annotations) if !bytes.Equal(expAnnotations, tsAnnotations) { diff --git a/e2e/k8s/eso.scoped.values.yaml b/e2e/k8s/eso.scoped.values.yaml new file mode 100644 index 000000000..cfe52f1ce --- /dev/null +++ b/e2e/k8s/eso.scoped.values.yaml @@ -0,0 +1,12 @@ +installCRDs: false +image: + repository: local/external-secrets + tag: test +scopedNamespace: test +extraEnv: + - name: AWS_SECRETSMANAGER_ENDPOINT + value: "http://localstack.default" + - name: AWS_STS_ENDPOINT + value: "http://localstack.default" + - name: AWS_SSM_ENDPOINT + value: "http://localstack.default" diff --git a/e2e/run.sh b/e2e/run.sh index b31121fa7..429cc3e05 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -58,5 +58,12 @@ kubectl run --rm \ --env="AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET:-}" \ --env="TENANT_ID=${TENANT_ID:-}" \ --env="VAULT_URL=${VAULT_URL:-}" \ + --env="GITLAB_TOKEN=${GITLAB_TOKEN:-}" \ + --env="GITLAB_PROJECT_ID=${GITLAB_PROJECT_ID:-}" \ + --env="ORACLE_USER_OCID=${ORACLE_USER_OCID:-}" \ + --env="ORACLE_TENANCY_OCID=${ORACLE_TENANCY_OCID:-}" \ + --env="ORACLE_REGION=${ORACLE_REGION:-}" \ + --env="ORACLE_FINGERPRINT=${ORACLE_FINGERPRINT:-}" \ + --env="ORACLE_KEY=${ORACLE_KEY:-}" \ --overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "external-secrets-e2e"}}' \ e2e --image=local/external-secrets-e2e:test diff --git a/e2e/suite/alibaba/alibaba.go b/e2e/suite/alibaba/alibaba.go new file mode 100644 index 000000000..d32e18b7c --- /dev/null +++ b/e2e/suite/alibaba/alibaba.go @@ -0,0 +1,47 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package alibaba + +import ( + "os" + + // nolint + . "github.com/onsi/ginkgo" + // nolint + . "github.com/onsi/ginkgo/extensions/table" + + "github.com/external-secrets/external-secrets/e2e/framework" + "github.com/external-secrets/external-secrets/e2e/suite/common" +) + +var _ = Describe("[alibaba] ", func() { + f := framework.New("eso-alibaba") + accessKeyID := os.Getenv("ACCESS_KEY_ID") + accessKeySecret := os.Getenv("ACCESS_KEY_SECRET") + regionID := os.Getenv("REGION_ID") + prov := newAlibabaProvider(f, accessKeyID, accessKeySecret, regionID) + + DescribeTable("sync secrets", framework.TableFunc(f, prov), + Entry(common.SimpleDataSync(f)), + Entry(common.NestedJSONWithGJSON(f)), + Entry(common.JSONDataFromSync(f)), + Entry(common.JSONDataWithProperty(f)), + Entry(common.JSONDataWithTemplate(f)), + Entry(common.DockerJSONConfig(f)), + Entry(common.DataPropertyDockerconfigJSON(f)), + Entry(common.SSHKeySync(f)), + Entry(common.SSHKeySyncDataProperty(f)), + ) +}) diff --git a/e2e/suite/alibaba/provider.go b/e2e/suite/alibaba/provider.go new file mode 100644 index 000000000..64bc97eab --- /dev/null +++ b/e2e/suite/alibaba/provider.go @@ -0,0 +1,118 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package alibaba + +import ( + "context" + + "github.com/aliyun/alibaba-cloud-sdk-go/services/kms" + + //nolint + . "github.com/onsi/ginkgo" + + //nolint + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" + esmeta "github.com/external-secrets/external-secrets/apis/meta/v1" + "github.com/external-secrets/external-secrets/e2e/framework" +) + +type alibabaProvider struct { + accessKeyID string + accessKeySecret string + regionID string + framework *framework.Framework +} + +const ( + secretName = "secretName" +) + +func newAlibabaProvider(f *framework.Framework, accessKeyID, accessKeySecret, regionID string) *alibabaProvider { + prov := &alibabaProvider{ + accessKeyID: accessKeyID, + accessKeySecret: accessKeySecret, + regionID: regionID, + framework: f, + } + BeforeEach(prov.BeforeEach) + return prov +} + +// CreateSecret creates a secret in both kv v1 and v2 provider. +func (s *alibabaProvider) CreateSecret(key, val string) { + client, err := kms.NewClientWithAccessKey(s.regionID, s.accessKeyID, s.accessKeySecret) + Expect(err).ToNot(HaveOccurred()) + kmssecretrequest := kms.CreateCreateSecretRequest() + kmssecretrequest.SecretName = secretName + kmssecretrequest.SecretData = "value" + _, err = client.CreateSecret(kmssecretrequest) + Expect(err).ToNot(HaveOccurred()) +} + +func (s *alibabaProvider) DeleteSecret(key string) { + client, err := kms.NewClientWithAccessKey(s.regionID, s.accessKeyID, s.accessKeySecret) + Expect(err).ToNot(HaveOccurred()) + kmssecretrequest := kms.CreateDeleteSecretRequest() + kmssecretrequest.SecretName = secretName + _, err = client.DeleteSecret(kmssecretrequest) + Expect(err).ToNot(HaveOccurred()) +} + +func (s *alibabaProvider) BeforeEach() { + // Creating an Alibaba secret + alibabaCreds := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: s.framework.Namespace.Name, + }, + StringData: map[string]string{ + secretName: "value", + }, + } + err := s.framework.CRClient.Create(context.Background(), alibabaCreds) + Expect(err).ToNot(HaveOccurred()) + + // Creating Alibaba secret store + secretStore := &esv1alpha1.SecretStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: s.framework.Namespace.Name, + Namespace: s.framework.Namespace.Name, + }, + Spec: esv1alpha1.SecretStoreSpec{ + Provider: &esv1alpha1.SecretStoreProvider{ + Alibaba: &esv1alpha1.AlibabaProvider{ + Auth: &esv1alpha1.AlibabaAuth{ + SecretRef: esv1alpha1.AlibabaAuthSecretRef{ + AccessKeyID: esmeta.SecretKeySelector{ + Name: "kms-secret", + Key: "keyid", + }, + AccessKeySecret: esmeta.SecretKeySelector{ + Name: "kms-secret", + Key: "accesskey", + }, + }, + }, + }, + }, + }, + } + err = s.framework.CRClient.Create(context.Background(), secretStore) + Expect(err).ToNot(HaveOccurred()) +} diff --git a/e2e/suite/aws/provider.go b/e2e/suite/aws/provider.go index 54313f07e..2357ed66c 100644 --- a/e2e/suite/aws/provider.go +++ b/e2e/suite/aws/provider.go @@ -11,6 +11,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package aws import ( diff --git a/e2e/suite/aws/secretsmanager.go b/e2e/suite/aws/secretsmanager.go index 60b984c58..e637963a0 100644 --- a/e2e/suite/aws/secretsmanager.go +++ b/e2e/suite/aws/secretsmanager.go @@ -11,6 +11,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package aws import ( diff --git a/e2e/suite/gitlab/gitlab.go b/e2e/suite/gitlab/gitlab.go new file mode 100644 index 000000000..2a795912b --- /dev/null +++ b/e2e/suite/gitlab/gitlab.go @@ -0,0 +1,45 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +limitations under the License. +*/ +package gitlab + +// TODO - Gitlab only accepts variable names with alphanumeric and '_' +// whereas ESO only accepts names with alphanumeric and '-'. +// Current workaround is to remove all hyphens and underscores set in e2e/framework/util/util.go +// and in e2e/suite/common/common.go, but this breaks Azure provider. + +import ( + "os" + + // nolint + . "github.com/onsi/ginkgo" + // nolint + . "github.com/onsi/ginkgo/extensions/table" + + "github.com/external-secrets/external-secrets/e2e/framework" + "github.com/external-secrets/external-secrets/e2e/suite/common" +) + +var _ = Describe("[gitlab] ", func() { + f := framework.New("esogitlab") + credentials := os.Getenv("GITLAB_TOKEN") + projectID := os.Getenv("GITLAB_PROJECT_ID") + prov := newGitlabProvider(f, credentials, projectID) + + DescribeTable("sync secrets", framework.TableFunc(f, prov), + Entry(common.SimpleDataSync(f)), + Entry(common.JSONDataWithProperty(f)), + Entry(common.JSONDataFromSync(f)), + Entry(common.NestedJSONWithGJSON(f)), + Entry(common.JSONDataWithTemplate(f)), + ) +}) diff --git a/e2e/suite/gitlab/provider.go b/e2e/suite/gitlab/provider.go new file mode 100644 index 000000000..deff8c6ff --- /dev/null +++ b/e2e/suite/gitlab/provider.go @@ -0,0 +1,131 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package gitlab + +import ( + "context" + "strings" + + // nolint + . "github.com/onsi/ginkgo" + + // nolint + . "github.com/onsi/gomega" + gitlab "github.com/xanzy/go-gitlab" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" + esmeta "github.com/external-secrets/external-secrets/apis/meta/v1" + "github.com/external-secrets/external-secrets/e2e/framework" +) + +type gitlabProvider struct { + credentials string + projectID string + framework *framework.Framework +} + +func newGitlabProvider(f *framework.Framework, credentials, projectID string) *gitlabProvider { + prov := &gitlabProvider{ + credentials: credentials, + projectID: projectID, + framework: f, + } + BeforeEach(prov.BeforeEach) + return prov +} + +func (s *gitlabProvider) CreateSecret(key, val string) { + // **Open the client + client, err := gitlab.NewClient(s.credentials) + Expect(err).ToNot(HaveOccurred()) + // Open the client** + + // Set variable options + variableKey := strings.ReplaceAll(key, "-", "_") + variableValue := val + + opt := gitlab.CreateProjectVariableOptions{ + Key: &variableKey, + Value: &variableValue, + VariableType: nil, + Protected: nil, + Masked: nil, + EnvironmentScope: nil, + } + + // Create a variable + _, _, err = client.ProjectVariables.CreateVariable(s.projectID, &opt) + + Expect(err).ToNot(HaveOccurred()) + // Versions aren't supported by Gitlab, but we could add + // more parameters to test +} + +func (s *gitlabProvider) DeleteSecret(key string) { + // **Open a client + client, err := gitlab.NewClient(s.credentials) + Expect(err).ToNot(HaveOccurred()) + // Open a client** + + // Delete the secret + _, err = client.ProjectVariables.RemoveVariable(s.projectID, strings.ReplaceAll(key, "-", "_")) + Expect(err).ToNot(HaveOccurred()) +} + +func (s *gitlabProvider) BeforeEach() { + By("creating a gitlab variable") + gitlabCreds := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "provider-secret", + Namespace: s.framework.Namespace.Name, + }, + // Puts access token into StringData + + StringData: map[string]string{ + "token": s.credentials, + "projectID": s.projectID, + }, + } + err := s.framework.CRClient.Create(context.Background(), gitlabCreds) + Expect(err).ToNot(HaveOccurred()) + + // Create a secret store - change these values to match YAML + By("creating a secret store for credentials") + secretStore := &esv1alpha1.SecretStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: s.framework.Namespace.Name, + Namespace: s.framework.Namespace.Name, + }, + Spec: esv1alpha1.SecretStoreSpec{ + Provider: &esv1alpha1.SecretStoreProvider{ + Gitlab: &esv1alpha1.GitlabProvider{ + ProjectID: s.projectID, + Auth: esv1alpha1.GitlabAuth{ + SecretRef: esv1alpha1.GitlabSecretRef{ + AccessToken: esmeta.SecretKeySelector{ + Name: "provider-secret", + Key: "token", + }, + }, + }, + }, + }, + }, + } + + err = s.framework.CRClient.Create(context.Background(), secretStore) + Expect(err).ToNot(HaveOccurred()) +} diff --git a/e2e/suite/oracle/oracle.go b/e2e/suite/oracle/oracle.go new file mode 100644 index 000000000..0a7dd8196 --- /dev/null +++ b/e2e/suite/oracle/oracle.go @@ -0,0 +1,47 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +limitations under the License. +*/ +package oracle + +import ( + "os" + + // nolint + . "github.com/onsi/ginkgo" + // nolint + . "github.com/onsi/ginkgo/extensions/table" + + "github.com/external-secrets/external-secrets/e2e/framework" + "github.com/external-secrets/external-secrets/e2e/suite/common" +) + +var _ = Describe("[oracle] ", func() { + f := framework.New("eso-oracle") + tenancy := os.Getenv("OCI_TENANCY_OCID") + user := os.Getenv("OCI_USER_OCID") + region := os.Getenv("OCI_REGION") + fingerprint := os.Getenv("OCI_FINGERPRINT") + privateKey := os.Getenv("OCI_PRIVATE_KEY") + prov := newOracleProvider(f, tenancy, user, region, fingerprint, privateKey) + + DescribeTable("sync secrets", framework.TableFunc(f, prov), + Entry(common.SimpleDataSync(f)), + Entry(common.NestedJSONWithGJSON(f)), + Entry(common.JSONDataFromSync(f)), + Entry(common.JSONDataWithProperty(f)), + Entry(common.JSONDataWithTemplate(f)), + Entry(common.DockerJSONConfig(f)), + Entry(common.DataPropertyDockerconfigJSON(f)), + Entry(common.SSHKeySync(f)), + Entry(common.SSHKeySyncDataProperty(f)), + ) +}) diff --git a/e2e/suite/oracle/provider.go b/e2e/suite/oracle/provider.go new file mode 100644 index 000000000..243ca3e5a --- /dev/null +++ b/e2e/suite/oracle/provider.go @@ -0,0 +1,124 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +limitations under the License. +*/ +package oracle + +import ( + "context" + + // nolint + . "github.com/onsi/ginkgo" + + // nolint + . "github.com/onsi/gomega" + "github.com/oracle/oci-go-sdk/v45/common" + vault "github.com/oracle/oci-go-sdk/v45/vault" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilpointer "k8s.io/utils/pointer" + + esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" + esmeta "github.com/external-secrets/external-secrets/apis/meta/v1" + "github.com/external-secrets/external-secrets/e2e/framework" +) + +type oracleProvider struct { + tenancy string + user string + region string + fingerprint string + privateKey string + framework *framework.Framework + ctx context.Context +} + +const ( + secretName = "secretName" +) + +func newOracleProvider(f *framework.Framework, tenancy, user, region, fingerprint, privateKey string) *oracleProvider { + prov := &oracleProvider{ + tenancy: tenancy, + user: user, + region: region, + fingerprint: fingerprint, + privateKey: privateKey, + framework: f, + } + BeforeEach(prov.BeforeEach) + return prov +} + +func (p *oracleProvider) CreateSecret(key, val string) { + configurationProvider := common.NewRawConfigurationProvider(p.tenancy, p.user, p.region, p.fingerprint, p.privateKey, nil) + client, err := vault.NewVaultsClientWithConfigurationProvider(configurationProvider) + Expect(err).ToNot(HaveOccurred()) + vmssecretrequest := vault.CreateSecretRequest{} + vmssecretrequest.SecretName = utilpointer.StringPtr(secretName) + vmssecretrequest.SecretContent = vault.Base64SecretContentDetails{ + Name: utilpointer.StringPtr(key), + Content: utilpointer.StringPtr(val), + } + _, err = client.CreateSecret(p.ctx, vmssecretrequest) + Expect(err).ToNot(HaveOccurred()) +} + +func (p *oracleProvider) DeleteSecret(key string) { + configurationProvider := common.NewRawConfigurationProvider(p.tenancy, p.user, p.region, p.fingerprint, p.privateKey, nil) + client, err := vault.NewVaultsClientWithConfigurationProvider(configurationProvider) + Expect(err).ToNot(HaveOccurred()) + vmssecretrequest := vault.ScheduleSecretDeletionRequest{} + vmssecretrequest.SecretId = utilpointer.StringPtr(key) + _, err = client.ScheduleSecretDeletion(p.ctx, vmssecretrequest) + Expect(err).ToNot(HaveOccurred()) +} + +func (p *oracleProvider) BeforeEach() { + OracleCreds := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: p.framework.Namespace.Name, + }, + StringData: map[string]string{ + secretName: "value", + }, + } + err := p.framework.CRClient.Create(context.Background(), OracleCreds) + Expect(err).ToNot(HaveOccurred()) + + secretStore := &esv1alpha1.SecretStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: p.framework.Namespace.Name, + Namespace: p.framework.Namespace.Name, + }, + Spec: esv1alpha1.SecretStoreSpec{ + Provider: &esv1alpha1.SecretStoreProvider{ + Oracle: &esv1alpha1.OracleProvider{ + Auth: esv1alpha1.OracleAuth{ + SecretRef: esv1alpha1.OracleSecretRef{ + Fingerprint: esmeta.SecretKeySelector{ + Name: "vms-secret", + Key: "keyid", + }, + PrivateKey: esmeta.SecretKeySelector{ + Name: "vms-secret", + Key: "accesskey", + }, + }, + }, + }, + }, + }, + } + err = p.framework.CRClient.Create(context.Background(), secretStore) + Expect(err).ToNot(HaveOccurred()) +} diff --git a/go.mod b/go.mod index b4c236d10..b29e2fa73 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,9 @@ module github.com/external-secrets/external-secrets go 1.16 replace ( + github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1 => ./apis/externalsecrets/v1alpha1 + github.com/external-secrets/external-secrets/pkg/provider/gitlab => ./pkg/provider/gitlab google.golang.org/grpc => google.golang.org/grpc v1.27.0 - k8s.io/api => k8s.io/api v0.21.2 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.21.2 k8s.io/apimachinery => k8s.io/apimachinery v0.21.2 @@ -39,6 +40,7 @@ require ( github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/IBM/go-sdk-core/v5 v5.5.0 github.com/IBM/secrets-manager-go-sdk v1.0.23 + github.com/aliyun/alibaba-cloud-sdk-go v1.61.1192 github.com/aws/aws-sdk-go v1.38.6 github.com/crossplane/crossplane-runtime v0.13.0 github.com/fatih/color v1.10.0 // indirect @@ -51,19 +53,22 @@ require ( github.com/googleapis/gax-go v1.0.3 github.com/hashicorp/go-hclog v0.14.1 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/go-retryablehttp v0.6.7 // indirect github.com/hashicorp/hcl v1.0.1-vault // indirect github.com/hashicorp/vault/api v1.0.5-0.20210224012239-b540be4b7ec4 github.com/kr/pretty v0.2.1 // indirect github.com/lestrrat-go/jwx v1.2.1 github.com/onsi/ginkgo v1.16.4 - github.com/onsi/gomega v1.13.0 + github.com/onsi/gomega v1.16.0 + github.com/oracle/oci-go-sdk/v45 v45.2.0 github.com/pierrec/lz4 v2.5.2+incompatible // indirect github.com/prometheus/client_golang v1.11.0 github.com/prometheus/client_model v0.2.0 github.com/spf13/cobra v1.1.3 // indirect github.com/stretchr/testify v1.7.0 github.com/tidwall/gjson v1.7.5 + github.com/xanzy/go-gitlab v0.50.1 + github.com/yandex-cloud/go-genproto v0.0.0-20210809082946-a97da516c588 + github.com/yandex-cloud/go-sdk v0.0.0-20210809100642-c13c40a429fa github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a go.uber.org/zap v1.17.0 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 @@ -72,12 +77,13 @@ require ( golang.org/x/tools v0.1.2-0.20210512205948-8287d5da45e4 // indirect google.golang.org/api v0.30.0 google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a + google.golang.org/grpc v1.31.0 honnef.co/go/tools v0.1.4 // indirect - k8s.io/api v0.21.2 - k8s.io/apimachinery v0.21.2 + k8s.io/api v0.21.3 + k8s.io/apimachinery v0.21.3 k8s.io/client-go v0.21.2 k8s.io/utils v0.0.0-20210527160623-6fdb442a123b - sigs.k8s.io/controller-runtime v0.9.2 + sigs.k8s.io/controller-runtime v0.9.3 sigs.k8s.io/controller-tools v0.5.0 software.sslmate.com/src/go-pkcs12 v0.0.0-20210415151418-c5206de65a78 ) diff --git a/go.sum b/go.sum index e96696710..a95d79455 100644 --- a/go.sum +++ b/go.sum @@ -77,6 +77,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/aliyun/alibaba-cloud-sdk-go v1.61.1192 h1:rRuMCkcoxoQ/kWSBN190JmD292PrYnpl7KyRWhYrjnY= +github.com/aliyun/alibaba-cloud-sdk-go v1.61.1192/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -98,6 +100,7 @@ github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -135,6 +138,7 @@ github.com/decred/dcrd/chaincfg/chainhash v1.0.2/go.mod h1:BpbrGgrPTr3YJYRN3Bm+D github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v3 v3.0.0 h1:sgNeV1VRMDzs6rzyPpxyM0jp317hnwiq58Filgag2xw= github.com/decred/dcrd/dcrec/secp256k1/v3 v3.0.0/go.mod h1:J70FGZSbzsjecRTiTzER+3f1KZLNaXkuv+yeFTKoxM8= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= @@ -166,6 +170,7 @@ github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -242,6 +247,7 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= @@ -291,6 +297,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= @@ -320,6 +328,7 @@ github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3i github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -352,8 +361,8 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9 github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= -github.com/hashicorp/go-retryablehttp v0.6.7 h1:8/CAEZt/+F7kR7GevNHulKkUjLht3CPmn7egmhieNKo= -github.com/hashicorp/go-retryablehttp v0.6.7/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs= +github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= @@ -401,6 +410,7 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -408,6 +418,7 @@ github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMW github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= @@ -479,6 +490,7 @@ github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -530,8 +542,11 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= -github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= +github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/oracle/oci-go-sdk/v45 v45.2.0 h1:vCPoQlE+DOrM2heJn66rvPU6fbsc/0Cxtzs2jnFut6U= +github.com/oracle/oci-go-sdk/v45 v45.2.0/go.mod h1:ZM6LGiRO5TPQJxTlrXbcHMbClE775wnGD5U/EerCsRw= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -594,7 +609,10 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -640,11 +658,17 @@ github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqri github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/xanzy/go-gitlab v0.50.1 h1:eH1G0/ZV1j81rhGrtbcePjbM5Ern7mPA4Xjt+yE+2PQ= +github.com/xanzy/go-gitlab v0.50.1/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yandex-cloud/go-genproto v0.0.0-20210809082946-a97da516c588 h1:Lbz8X5Nre0Lg5QgCblmo0AhScWxeN3CVnX+mZ5Hxksk= +github.com/yandex-cloud/go-genproto v0.0.0-20210809082946-a97da516c588/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE= +github.com/yandex-cloud/go-sdk v0.0.0-20210809100642-c13c40a429fa h1:Un1jWl/YWbK1179aMbsEZ6uLlDjjBAjL8KXldho1Umo= +github.com/yandex-cloud/go-sdk v0.0.0-20210809100642-c13c40a429fa/go.mod h1:UkgAKjyQo+Pylt2HTYz/G0PgnxmKOJ9IX/3XiRYQ9Ns= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= @@ -763,6 +787,7 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200320220750-118fecf932d8/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -781,6 +806,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -969,6 +995,7 @@ google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0 h1:yfrXXP61wVuLb0vBcG6qaOoIoqYEzOQS8jum51jkv2w= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= @@ -996,6 +1023,7 @@ google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200323114720-3f67cca34472/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -1039,6 +1067,8 @@ gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+a gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -1104,8 +1134,8 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-runtime v0.8.0/go.mod h1:v9Lbj5oX443uR7GXYY46E0EE2o7k2YxQ58GxVNeXSW4= -sigs.k8s.io/controller-runtime v0.9.2 h1:MnCAsopQno6+hI9SgJHKddzXpmv2wtouZz6931Eax+Q= -sigs.k8s.io/controller-runtime v0.9.2/go.mod h1:TxzMCHyEUpaeuOiZx/bIdc2T81vfs/aKdvJt9wuu0zk= +sigs.k8s.io/controller-runtime v0.9.3 h1:n075bHQ1wb8hpX7C27pNrqsb0fj8mcfCQfNX+oKTbYE= +sigs.k8s.io/controller-runtime v0.9.3/go.mod h1:TxzMCHyEUpaeuOiZx/bIdc2T81vfs/aKdvJt9wuu0zk= sigs.k8s.io/controller-tools v0.2.4/go.mod h1:m/ztfQNocGYBgTTCmFdnK94uVvgxeZeE3LtJvd/jIzA= sigs.k8s.io/controller-tools v0.5.0 h1:3u2RCwOlp0cjCALAigpOcbAf50pE+kHSdueUosrC/AE= sigs.k8s.io/controller-tools v0.5.0/go.mod h1:JTsstrMpxs+9BUj6eGuAaEb6SDSPTeVtUyp0jmnAM/I= diff --git a/hack/api-docs/mkdocs.yml b/hack/api-docs/mkdocs.yml index ad3546ff0..32c47f0a0 100644 --- a/hack/api-docs/mkdocs.yml +++ b/hack/api-docs/mkdocs.yml @@ -46,6 +46,12 @@ nav: - IBM: - Secrets Manager: provider-ibm-secrets-manager.md - HashiCorp Vault: provider-hashicorp-vault.md + - Yandex: + - Lockbox: provider-yandex-lockbox.md + - Gitlab: + - Gitlab Project Variables: provider-gitlab-project-variables.md + - Oracle: + - Oracle Vault: provider-oracle-vault.md - References: - API specification: spec.md - Contributing: diff --git a/main.go b/main.go index 2598f64db..63763df15 100644 --- a/main.go +++ b/main.go @@ -46,18 +46,20 @@ func main() { var controllerClass string var enableLeaderElection bool var loglevel string + var namespace string flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&controllerClass, "controller-class", "default", "the controller is instantiated with a specific controller name and filters ES based on this property") flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") flag.StringVar(&loglevel, "loglevel", "info", "loglevel to use, one of: debug, info, warn, error, dpanic, panic, fatal") + flag.StringVar(&namespace, "namespace", "", "watch external secrets scoped in the provided namespace only") flag.Parse() var lvl zapcore.Level err := lvl.UnmarshalText([]byte(loglevel)) if err != nil { - setupLog.Error(err, "error unmarshaling loglevel") + setupLog.Error(err, "error unmarshalling loglevel") os.Exit(1) } logger := zap.New(zap.Level(lvl)) @@ -69,6 +71,7 @@ func main() { Port: 9443, LeaderElection: enableLeaderElection, LeaderElectionID: "external-secrets-controller", + Namespace: namespace, }) if err != nil { setupLog.Error(err, "unable to start manager") diff --git a/pkg/controllers/externalsecret/externalsecret_controller.go b/pkg/controllers/externalsecret/externalsecret_controller.go index 7f10c78b4..0bf318ed8 100644 --- a/pkg/controllers/externalsecret/externalsecret_controller.go +++ b/pkg/controllers/externalsecret/externalsecret_controller.go @@ -16,9 +16,6 @@ package externalsecret import ( "context" - - // nolint - "crypto/md5" "fmt" "time" @@ -38,9 +35,8 @@ import ( // Loading registered providers. _ "github.com/external-secrets/external-secrets/pkg/provider/register" - schema "github.com/external-secrets/external-secrets/pkg/provider/schema" - "github.com/external-secrets/external-secrets/pkg/template" - utils "github.com/external-secrets/external-secrets/pkg/utils" + "github.com/external-secrets/external-secrets/pkg/provider/schema" + "github.com/external-secrets/external-secrets/pkg/utils" ) const ( @@ -54,10 +50,12 @@ const ( errStoreRef = "could not get store reference" errStoreProvider = "could not get store provider" errStoreClient = "could not get provider client" + errGetExistingSecret = "could not get existing secret: %w" errCloseStoreClient = "could not close provider client" errSetCtrlReference = "could not set ExternalSecret controller reference: %w" errFetchTplFrom = "error fetching templateFrom data: %w" errGetSecretData = "could not get secret data from provider: %w" + errApplyTemplate = "could not apply template: %w" errExecTpl = "could not execute template: %w" errPolicyMergeNotFound = "the desired secret %s was not found. With creationPolicy=Merge the secret won't be created" errPolicyMergeGetSecret = "unable to get secret %s: %w" @@ -126,7 +124,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // check if store should be handled by this controller instance if !shouldProcessStore(store, r.ControllerClass) { - log.Info("skippig unmanaged store") + log.Info("skipping unmanaged store") return ctrl.Result{}, nil } @@ -147,7 +145,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } defer func() { - err = secretClient.Close() + err = secretClient.Close(ctx) if err != nil { log.Error(err, errCloseStoreClient) } @@ -158,11 +156,27 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu refreshInt = externalSecret.Spec.RefreshInterval.Duration } + // Target Secret Name should default to the ExternalSecret name if not explicitly specified + secretName := externalSecret.Spec.Target.Name + if secretName == "" { + secretName = externalSecret.ObjectMeta.Name + } + + // fetch external secret, we need to ensure that it exists, and it's hashmap corresponds + var existingSecret v1.Secret + err = r.Get(ctx, types.NamespacedName{ + Name: secretName, + Namespace: externalSecret.Namespace, + }, &existingSecret) + if err != nil && !apierrors.IsNotFound(err) { + log.Error(err, errGetExistingSecret) + } + // refresh should be skipped if // 1. resource generation hasn't changed // 2. refresh interval is 0 // 3. if we're still within refresh-interval - if !shouldRefresh(externalSecret) { + if !shouldRefresh(externalSecret) && isSecretValid(existingSecret) { log.V(1).Info("skipping refresh", "rv", getResourceVersion(externalSecret)) return ctrl.Result{RequeueAfter: refreshInt}, nil } @@ -176,7 +190,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu secret := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: externalSecret.Spec.Target.Name, + Name: secretName, Namespace: externalSecret.Namespace, }, Immutable: &externalSecret.Spec.Target.Immutable, @@ -190,43 +204,21 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return fmt.Errorf(errSetCtrlReference, err) } } - mergeMetadata(secret, externalSecret) - var tplMap map[string][]byte - var dataMap map[string][]byte - // get data - dataMap, err = r.getProviderSecretData(ctx, secretClient, &externalSecret) + dataMap, err := r.getProviderSecretData(ctx, secretClient, &externalSecret) if err != nil { return fmt.Errorf(errGetSecretData, err) } - // no template: copy data and return - if externalSecret.Spec.Target.Template == nil { - for k, v := range dataMap { - secret.Data[k] = v - } - return nil + err = r.applyTemplate(ctx, &externalSecret, secret, dataMap) + if err != nil { + return fmt.Errorf(errApplyTemplate, err) } - // template: fetch & execute templates - tplMap, err = r.getTemplateData(ctx, &externalSecret) - if err != nil { - return fmt.Errorf(errFetchTplFrom, err) - } - // override templateFrom data with template data - for k, v := range externalSecret.Spec.Target.Template.Data { - tplMap[k] = []byte(v) - } - - log.V(1).Info("found template data", "tpl_data", tplMap) - err = template.Execute(tplMap, dataMap, secret) - if err != nil { - return fmt.Errorf(errExecTpl, err) - } return nil } - //nolint + // nolint switch externalSecret.Spec.Target.CreationPolicy { case esv1alpha1.Merge: err = patchSecret(ctx, r.Client, r.Scheme, secret, mutationFunc) @@ -273,7 +265,7 @@ func patchSecret(ctx context.Context, c client.Client, scheme *runtime.Scheme, s // https://github.com/kubernetes-sigs/controller-runtime/issues/526 // https://github.com/kubernetes-sigs/controller-runtime/issues/1517 // https://github.com/kubernetes/kubernetes/issues/80609 - // we need to manually set it befor doing a Patch() as it depends on the GVK + // we need to manually set it before doing a Patch() as it depends on the GVK gvks, unversioned, err := scheme.ObjectKinds(secret) if err != nil { return err @@ -308,12 +300,10 @@ func hashMeta(m metav1.ObjectMeta) string { annotations map[string]string labels map[string]string } - h := md5.New() //nolint - _, _ = h.Write([]byte(fmt.Sprintf("%v", meta{ + return utils.ObjectHash(meta{ annotations: m.Annotations, labels: m.Labels, - }))) - return fmt.Sprintf("%x", h.Sum(nil)) + }) } func shouldRefresh(es esv1alpha1.ExternalSecret) bool { @@ -321,6 +311,7 @@ func shouldRefresh(es esv1alpha1.ExternalSecret) bool { if es.Status.SyncedResourceVersion != getResourceVersion(es) { return true } + // skip refresh if refresh interval is 0 if es.Spec.RefreshInterval.Duration == 0 && es.Status.SyncedResourceVersion != "" { return false @@ -339,7 +330,6 @@ func shouldReconcile(es esv1alpha1.ExternalSecret) bool { } func hasSyncedCondition(es esv1alpha1.ExternalSecret) bool { - for _, condition := range es.Status.Conditions { if condition.Reason == "SecretSynced" { return true @@ -348,24 +338,18 @@ func hasSyncedCondition(es esv1alpha1.ExternalSecret) bool { return false } -// we do not want to force-override the label/annotations -// and only copy the necessary key/value pairs. -func mergeMetadata(secret *v1.Secret, externalSecret esv1alpha1.ExternalSecret) { - if secret.ObjectMeta.Labels == nil { - secret.ObjectMeta.Labels = make(map[string]string) +// isSecretValid checks if the secret exists, and it's data is consistent with the calculated hash. +func isSecretValid(existingSecret v1.Secret) bool { + // if target secret doesn't exist, or annotations as not set, we need to refresh + if existingSecret.UID == "" || existingSecret.Annotations == nil { + return false } - if secret.ObjectMeta.Annotations == nil { - secret.ObjectMeta.Annotations = make(map[string]string) + + // if the calculated hash is different from the calculation, then it's invalid + if existingSecret.Annotations[esv1alpha1.AnnotationDataHash] != utils.ObjectHash(existingSecret.Data) { + return false } - if externalSecret.Spec.Target.Template == nil { - utils.MergeStringMap(secret.ObjectMeta.Labels, externalSecret.ObjectMeta.Labels) - utils.MergeStringMap(secret.ObjectMeta.Annotations, externalSecret.ObjectMeta.Annotations) - return - } - // if template is defined: use those labels/annotations - secret.Type = externalSecret.Spec.Target.Template.Type - utils.MergeStringMap(secret.ObjectMeta.Labels, externalSecret.Spec.Target.Template.Metadata.Labels) - utils.MergeStringMap(secret.ObjectMeta.Annotations, externalSecret.Spec.Target.Template.Metadata.Annotations) + return true } // getStore returns the store with the provided ExternalSecret. @@ -419,50 +403,6 @@ func (r *Reconciler) getProviderSecretData(ctx context.Context, providerClient p return providerData, nil } -func (r *Reconciler) getTemplateData(ctx context.Context, externalSecret *esv1alpha1.ExternalSecret) (map[string][]byte, error) { - out := make(map[string][]byte) - if externalSecret.Spec.Target.Template == nil { - return out, nil - } - for _, tpl := range externalSecret.Spec.Target.Template.TemplateFrom { - if tpl.ConfigMap != nil { - var cm v1.ConfigMap - err := r.Client.Get(ctx, types.NamespacedName{ - Name: tpl.ConfigMap.Name, - Namespace: externalSecret.Namespace, - }, &cm) - if err != nil { - return nil, err - } - for _, k := range tpl.ConfigMap.Items { - val, ok := cm.Data[k.Key] - if !ok { - return nil, fmt.Errorf(errTplCMMissingKey, tpl.ConfigMap.Name, k.Key) - } - out[k.Key] = []byte(val) - } - } - if tpl.Secret != nil { - var sec v1.Secret - err := r.Client.Get(ctx, types.NamespacedName{ - Name: tpl.Secret.Name, - Namespace: externalSecret.Namespace, - }, &sec) - if err != nil { - return nil, err - } - for _, k := range tpl.Secret.Items { - val, ok := sec.Data[k.Key] - if !ok { - return nil, fmt.Errorf(errTplSecMissingKey, tpl.Secret.Name, k.Key) - } - out[k.Key] = val - } - } - } - return out, nil -} - // SetupWithManager returns a new controller builder that will be started by the provided Manager. func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). diff --git a/pkg/controllers/externalsecret/externalsecret_controller_template.go b/pkg/controllers/externalsecret/externalsecret_controller_template.go new file mode 100644 index 000000000..b20e416e4 --- /dev/null +++ b/pkg/controllers/externalsecret/externalsecret_controller_template.go @@ -0,0 +1,157 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package externalsecret + +import ( + "context" + "fmt" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" + + // Loading registered providers. + _ "github.com/external-secrets/external-secrets/pkg/provider/register" + "github.com/external-secrets/external-secrets/pkg/template" + utils "github.com/external-secrets/external-secrets/pkg/utils" +) + +// merge template in the following order: +// * template.Data (highest precedence) +// * template.templateFrom +// * secret via es.data or es.dataFrom. +func (r *Reconciler) applyTemplate(ctx context.Context, es *esv1alpha1.ExternalSecret, secret *v1.Secret, dataMap map[string][]byte) error { + mergeMetadata(secret, es) + + // no template: copy data and return + if es.Spec.Target.Template == nil { + secret.Data = dataMap + secret.Annotations[esv1alpha1.AnnotationDataHash] = utils.ObjectHash(secret.Data) + return nil + } + + // fetch templates defined in template.templateFrom + tplMap, err := r.getTemplateData(ctx, es) + if err != nil { + return fmt.Errorf(errFetchTplFrom, err) + } + + // explicitly defined template.Data takes precedence over templateFrom + for k, v := range es.Spec.Target.Template.Data { + tplMap[k] = []byte(v) + } + r.Log.V(1).Info("found template data", "tpl_data", tplMap) + + err = template.Execute(tplMap, dataMap, secret) + if err != nil { + return fmt.Errorf(errExecTpl, err) + } + + // if no data was provided by template fallback + // to value from the provider + if len(es.Spec.Target.Template.Data) == 0 { + for k, v := range dataMap { + secret.Data[k] = v + } + } + secret.Annotations[esv1alpha1.AnnotationDataHash] = utils.ObjectHash(secret.Data) + + return nil +} + +// we do not want to force-override the label/annotations +// and only copy the necessary key/value pairs. +func mergeMetadata(secret *v1.Secret, externalSecret *esv1alpha1.ExternalSecret) { + if secret.ObjectMeta.Labels == nil { + secret.ObjectMeta.Labels = make(map[string]string) + } + if secret.ObjectMeta.Annotations == nil { + secret.ObjectMeta.Annotations = make(map[string]string) + } + if externalSecret.Spec.Target.Template == nil { + utils.MergeStringMap(secret.ObjectMeta.Labels, externalSecret.ObjectMeta.Labels) + utils.MergeStringMap(secret.ObjectMeta.Annotations, externalSecret.ObjectMeta.Annotations) + return + } + // if template is defined: use those labels/annotations + secret.Type = externalSecret.Spec.Target.Template.Type + utils.MergeStringMap(secret.ObjectMeta.Labels, externalSecret.Spec.Target.Template.Metadata.Labels) + utils.MergeStringMap(secret.ObjectMeta.Annotations, externalSecret.Spec.Target.Template.Metadata.Annotations) +} + +func (r *Reconciler) getTemplateData(ctx context.Context, externalSecret *esv1alpha1.ExternalSecret) (map[string][]byte, error) { + out := make(map[string][]byte) + if externalSecret.Spec.Target.Template == nil { + return out, nil + } + for _, tpl := range externalSecret.Spec.Target.Template.TemplateFrom { + err := mergeConfigMap(ctx, r.Client, externalSecret, tpl, out) + if err != nil { + return nil, err + } + err = mergeSecret(ctx, r.Client, externalSecret, tpl, out) + if err != nil { + return nil, err + } + } + return out, nil +} + +func mergeConfigMap(ctx context.Context, k8sClient client.Client, es *esv1alpha1.ExternalSecret, tpl esv1alpha1.TemplateFrom, out map[string][]byte) error { + if tpl.ConfigMap == nil { + return nil + } + + var cm v1.ConfigMap + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: tpl.ConfigMap.Name, + Namespace: es.Namespace, + }, &cm) + if err != nil { + return err + } + for _, k := range tpl.ConfigMap.Items { + val, ok := cm.Data[k.Key] + if !ok { + return fmt.Errorf(errTplCMMissingKey, tpl.ConfigMap.Name, k.Key) + } + out[k.Key] = []byte(val) + } + return nil +} + +func mergeSecret(ctx context.Context, k8sClient client.Client, es *esv1alpha1.ExternalSecret, tpl esv1alpha1.TemplateFrom, out map[string][]byte) error { + if tpl.Secret == nil { + return nil + } + var sec v1.Secret + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: tpl.Secret.Name, + Namespace: es.Namespace, + }, &sec) + if err != nil { + return err + } + for _, k := range tpl.Secret.Items { + val, ok := sec.Data[k.Key] + if !ok { + return fmt.Errorf(errTplSecMissingKey, tpl.Secret.Name, k.Key) + } + out[k.Key] = val + } + return nil +} diff --git a/pkg/controllers/externalsecret/externalsecret_controller_test.go b/pkg/controllers/externalsecret/externalsecret_controller_test.go index cb05429c0..cdfeea1b7 100644 --- a/pkg/controllers/externalsecret/externalsecret_controller_test.go +++ b/pkg/controllers/externalsecret/externalsecret_controller_test.go @@ -16,6 +16,8 @@ package externalsecret import ( "context" "fmt" + "os" + "strconv" "time" . "github.com/onsi/ginkgo" @@ -59,6 +61,74 @@ type testCase struct { type testTweaks func(*testCase) +var _ = Describe("Kind=secret existence logic", func() { + type testCase struct { + Name string + Input v1.Secret + ExpectedOutput bool + } + tests := []testCase{ + { + Name: "Should not be valid in case of missing uid", + Input: v1.Secret{}, + ExpectedOutput: false, + }, + { + Name: "A nil annotation should not be valid", + Input: v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + UID: "xxx", + Annotations: map[string]string{}, + }, + }, + ExpectedOutput: false, + }, + { + Name: "A nil annotation should not be valid", + Input: v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + UID: "xxx", + Annotations: map[string]string{}, + }, + }, + ExpectedOutput: false, + }, + { + Name: "An invalid annotation hash should not be valid", + Input: v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + UID: "xxx", + Annotations: map[string]string{ + esv1alpha1.AnnotationDataHash: "xxxxxx", + }, + }, + }, + ExpectedOutput: false, + }, + { + Name: "A valid config map should return true", + Input: v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + UID: "xxx", + Annotations: map[string]string{ + esv1alpha1.AnnotationDataHash: "caa0155759a6a9b3b6ada5a6883ee2bb", + }, + }, + Data: map[string][]byte{ + "foo": []byte("value1"), + "bar": []byte("value2"), + }, + }, + ExpectedOutput: true, + }, + } + + for _, tt := range tests { + It(tt.Name, func() { + Expect(isSecretValid(tt.Input)).To(BeEquivalentTo(tt.ExpectedOutput)) + }) + } +}) var _ = Describe("ExternalSecret controller", func() { const ( ExternalSecretName = "test-es" @@ -68,6 +138,13 @@ var _ = Describe("ExternalSecret controller", func() { var ExternalSecretNamespace string + // if we are in debug and need to increase the timeout for testing, we can do so by using an env var + if customTimeout := os.Getenv("TEST_CUSTOM_TIMEOUT_SEC"); customTimeout != "" { + if t, err := strconv.Atoi(customTimeout); err == nil { + timeout = time.Second * time.Duration(t) + } + } + BeforeEach(func() { var err error ExternalSecretNamespace, err = CreateNamespace("test-ns", k8sClient) @@ -157,6 +234,23 @@ var _ = Describe("ExternalSecret controller", func() { "hihihih": "hehehe", } fakeProvider.WithGetSecret([]byte(secretVal), nil) + tc.checkSecret = func(es *esv1alpha1.ExternalSecret, secret *v1.Secret) { + // check value + Expect(string(secret.Data[targetProp])).To(Equal(secretVal)) + + // check labels & annotations + Expect(secret.ObjectMeta.Labels).To(BeEquivalentTo(es.ObjectMeta.Labels)) + for k, v := range es.ObjectMeta.Annotations { + Expect(secret.ObjectMeta.Annotations).To(HaveKeyWithValue(k, v)) + } + // ownerRef must not not be set! + Expect(hasOwnerRef(secret.ObjectMeta, "ExternalSecret", ExternalSecretName)).To(BeTrue()) + } + } + + checkPrometheusCounters := func(tc *testCase) { + const secretVal = "someValue" + fakeProvider.WithGetSecret([]byte(secretVal), nil) tc.checkSecret = func(es *esv1alpha1.ExternalSecret, secret *v1.Secret) { Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1alpha1.ExternalSecretReady, v1.ConditionFalse, 0.0)).To(BeTrue()) Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1alpha1.ExternalSecretReady, v1.ConditionTrue, 1.0)).To(BeTrue()) @@ -164,15 +258,6 @@ var _ = Describe("ExternalSecret controller", func() { Expect(syncCallsTotal.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed()) return metric.GetCounter().GetValue() == 1.0 }, timeout, interval).Should(BeTrue()) - - // check value - Expect(string(secret.Data[targetProp])).To(Equal(secretVal)) - - // check labels & annotations - Expect(secret.ObjectMeta.Labels).To(BeEquivalentTo(es.ObjectMeta.Labels)) - Expect(secret.ObjectMeta.Annotations).To(BeEquivalentTo(es.ObjectMeta.Annotations)) - // ownerRef must not not be set! - Expect(hasOwnerRef(secret.ObjectMeta, "ExternalSecret", ExternalSecretName)).To(BeTrue()) } } @@ -198,23 +283,22 @@ var _ = Describe("ExternalSecret controller", func() { fakeProvider.WithGetSecret([]byte(secretVal), nil) tc.checkSecret = func(es *esv1alpha1.ExternalSecret, secret *v1.Secret) { - Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1alpha1.ExternalSecretReady, v1.ConditionFalse, 0.0)).To(BeTrue()) - Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1alpha1.ExternalSecretReady, v1.ConditionTrue, 1.0)).To(BeTrue()) - Eventually(func() bool { - Expect(syncCallsTotal.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed()) - return metric.GetCounter().GetValue() == 1.0 - }, timeout, interval).Should(BeTrue()) - // check value Expect(string(secret.Data[existingKey])).To(Equal(existingVal)) Expect(string(secret.Data[targetProp])).To(Equal(secretVal)) // check labels & annotations Expect(secret.ObjectMeta.Labels).To(BeEquivalentTo(es.ObjectMeta.Labels)) - Expect(secret.ObjectMeta.Annotations).To(BeEquivalentTo(es.ObjectMeta.Annotations)) + for k, v := range es.ObjectMeta.Annotations { + Expect(secret.ObjectMeta.Annotations).To(HaveKeyWithValue(k, v)) + } Expect(hasOwnerRef(secret.ObjectMeta, "ExternalSecret", ExternalSecretName)).To(BeFalse()) Expect(secret.ObjectMeta.ManagedFields).To(HaveLen(2)) - Expect(hasFieldOwnership(secret.ObjectMeta, "external-secrets", "{\"f:data\":{\"f:targetProperty\":{}},\"f:immutable\":{}}")).To(BeTrue()) + Expect(hasFieldOwnership( + secret.ObjectMeta, + "external-secrets", + fmt.Sprintf("{\"f:data\":{\"f:targetProperty\":{}},\"f:immutable\":{},\"f:metadata\":{\"f:annotations\":{\"f:%s\":{}}}}", esv1alpha1.AnnotationDataHash)), + ).To(BeTrue()) Expect(hasFieldOwnership(secret.ObjectMeta, "fake.manager", "{\"f:data\":{\".\":{},\"f:pre-existing-key\":{}},\"f:type\":{}}")).To(BeTrue()) } } @@ -313,20 +397,15 @@ var _ = Describe("ExternalSecret controller", func() { } fakeProvider.WithGetSecret([]byte(secretVal), nil) tc.checkSecret = func(es *esv1alpha1.ExternalSecret, secret *v1.Secret) { - Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1alpha1.ExternalSecretReady, v1.ConditionFalse, 0.0)).To(BeTrue()) - Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1alpha1.ExternalSecretReady, v1.ConditionTrue, 1.0)).To(BeTrue()) - Eventually(func() bool { - Expect(syncCallsTotal.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed()) - return metric.GetCounter().GetValue() == 1.0 - }, timeout, interval).Should(BeTrue()) - // check values Expect(string(secret.Data[targetProp])).To(Equal(expectedSecretVal)) Expect(string(secret.Data[tplStaticKey])).To(Equal(tplStaticVal)) // labels/annotations should be taken from the template Expect(secret.ObjectMeta.Labels).To(BeEquivalentTo(es.Spec.Target.Template.Metadata.Labels)) - Expect(secret.ObjectMeta.Annotations).To(BeEquivalentTo(es.Spec.Target.Template.Metadata.Annotations)) + for k, v := range es.Spec.Target.Template.Metadata.Annotations { + Expect(secret.ObjectMeta.Annotations).To(HaveKeyWithValue(k, v)) + } } } @@ -341,17 +420,29 @@ var _ = Describe("ExternalSecret controller", func() { const tplStaticKey = "tplstatickey" const tplStaticVal = "tplstaticvalue" const tplFromCMName = "template-cm" + const tplFromSecretName = "template-secret" const tplFromKey = "tpl-from-key" + const tplFromSecKey = "tpl-from-sec-key" const tplFromVal = "tpl-from-value: {{ .targetProperty | toString }} // {{ .bar | toString }}" + const tplFromSecVal = "tpl-from-sec-value: {{ .targetProperty | toString }} // {{ .bar | toString }}" Expect(k8sClient.Create(context.Background(), &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: "template-cm", + Name: tplFromCMName, Namespace: ExternalSecretNamespace, }, Data: map[string]string{ tplFromKey: tplFromVal, }, })).To(Succeed()) + Expect(k8sClient.Create(context.Background(), &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: tplFromSecretName, + Namespace: ExternalSecretNamespace, + }, + Data: map[string][]byte{ + tplFromSecKey: []byte(tplFromSecVal), + }, + })).To(Succeed()) tc.externalSecret.Spec.Target.Template = &esv1alpha1.ExternalSecretTemplate{ Metadata: esv1alpha1.ExternalSecretTemplateMetadata{}, Type: v1.SecretTypeOpaque, @@ -366,6 +457,16 @@ var _ = Describe("ExternalSecret controller", func() { }, }, }, + { + Secret: &esv1alpha1.TemplateRef{ + Name: tplFromSecretName, + Items: []esv1alpha1.TemplateRefItem{ + { + Key: tplFromSecKey, + }, + }, + }, + }, }, Data: map[string]string{ // this should be the data value, not dataFrom @@ -392,6 +493,7 @@ var _ = Describe("ExternalSecret controller", func() { Expect(string(secret.Data[tplStaticKey])).To(Equal(tplStaticVal)) Expect(string(secret.Data["bar"])).To(Equal("value from map: map-bar-value")) Expect(string(secret.Data[tplFromKey])).To(Equal("tpl-from-value: someValue // map-bar-value")) + Expect(string(secret.Data[tplFromSecKey])).To(Equal("tpl-from-sec-value: someValue // map-bar-value")) } } @@ -420,7 +522,12 @@ var _ = Describe("ExternalSecret controller", func() { // labels/annotations should be taken from the template Expect(secret.ObjectMeta.Labels).To(BeEquivalentTo(es.Spec.Target.Template.Metadata.Labels)) - Expect(secret.ObjectMeta.Annotations).To(BeEquivalentTo(es.Spec.Target.Template.Metadata.Annotations)) + + // a secret will always have some extra annotations (i.e. hashmap check), so we only check for specific + // source annotations + for k, v := range es.Spec.Target.Template.Metadata.Annotations { + Expect(secret.ObjectMeta.Annotations).To(HaveKeyWithValue(k, v)) + } cleanEs := tc.externalSecret.DeepCopy() @@ -447,7 +554,31 @@ var _ = Describe("ExternalSecret controller", func() { // also check labels/annotations have been updated Expect(secret.ObjectMeta.Labels).To(BeEquivalentTo(es.Spec.Target.Template.Metadata.Labels)) - Expect(secret.ObjectMeta.Annotations).To(BeEquivalentTo(es.Spec.Target.Template.Metadata.Annotations)) + for k, v := range es.Spec.Target.Template.Metadata.Annotations { + Expect(secret.ObjectMeta.Annotations).To(HaveKeyWithValue(k, v)) + } + } + } + + onlyMetadataFromTemplate := func(tc *testCase) { + const secretVal = "someValue" + tc.externalSecret.Spec.RefreshInterval = &metav1.Duration{Duration: time.Second} + tc.externalSecret.Spec.Target.Template = &esv1alpha1.ExternalSecretTemplate{ + Metadata: esv1alpha1.ExternalSecretTemplateMetadata{ + Labels: map[string]string{"foo": "bar"}, + Annotations: map[string]string{"foo": "bar"}, + }, + } + fakeProvider.WithGetSecret([]byte(secretVal), nil) + tc.checkSecret = func(es *esv1alpha1.ExternalSecret, secret *v1.Secret) { + // check values + Expect(string(secret.Data[targetProp])).To(Equal(secretVal)) + + // labels/annotations should be taken from the template + Expect(secret.ObjectMeta.Labels).To(BeEquivalentTo(es.Spec.Target.Template.Metadata.Labels)) + for k, v := range es.Spec.Target.Template.Metadata.Annotations { + Expect(secret.ObjectMeta.Annotations).To(HaveKeyWithValue(k, v)) + } } } @@ -459,13 +590,6 @@ var _ = Describe("ExternalSecret controller", func() { fakeProvider.WithGetSecret([]byte(secretVal), nil) tc.externalSecret.Spec.RefreshInterval = &metav1.Duration{Duration: time.Second} tc.checkSecret = func(es *esv1alpha1.ExternalSecret, secret *v1.Secret) { - Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1alpha1.ExternalSecretReady, v1.ConditionFalse, 0.0)).To(BeTrue()) - Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1alpha1.ExternalSecretReady, v1.ConditionTrue, 1.0)).To(BeTrue()) - Eventually(func() bool { - Expect(syncCallsTotal.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed()) - return metric.GetCounter().GetValue() == 1.0 - }, timeout, interval).Should(BeTrue()) - // check values Expect(string(secret.Data[targetProp])).To(Equal(secretVal)) @@ -494,13 +618,6 @@ var _ = Describe("ExternalSecret controller", func() { fakeProvider.WithGetSecret([]byte(secretVal), nil) tc.externalSecret.Spec.RefreshInterval = &metav1.Duration{Duration: 0} tc.checkSecret = func(es *esv1alpha1.ExternalSecret, secret *v1.Secret) { - Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1alpha1.ExternalSecretReady, v1.ConditionFalse, 0.0)).To(BeTrue()) - Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1alpha1.ExternalSecretReady, v1.ConditionTrue, 1.0)).To(BeTrue()) - Eventually(func() bool { - Expect(syncCallsTotal.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed()) - return metric.GetCounter().GetValue() == 1.0 - }, timeout, interval).Should(BeTrue()) - // check values Expect(string(secret.Data[targetProp])).To(Equal(secretVal)) @@ -537,19 +654,40 @@ var _ = Describe("ExternalSecret controller", func() { "bar": []byte("map-bar-value"), }, nil) tc.checkSecret = func(es *esv1alpha1.ExternalSecret, secret *v1.Secret) { - Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1alpha1.ExternalSecretReady, v1.ConditionFalse, 0.0)).To(BeTrue()) - Expect(externalSecretConditionShouldBe(ExternalSecretName, ExternalSecretNamespace, esv1alpha1.ExternalSecretReady, v1.ConditionTrue, 1.0)).To(BeTrue()) - Eventually(func() bool { - Expect(syncCallsTotal.WithLabelValues(ExternalSecretName, ExternalSecretNamespace).Write(&metric)).To(Succeed()) - return metric.GetCounter().GetValue() == 1.0 - }, timeout, interval).Should(BeTrue()) - // check values Expect(string(secret.Data["foo"])).To(Equal("map-foo-value")) Expect(string(secret.Data["bar"])).To(Equal("map-bar-value")) } } + // with dataFrom and using a template + // should be put into the secret + syncWithDataFromTemplate := func(tc *testCase) { + tc.externalSecret.Spec.Data = nil + tc.externalSecret.Spec.Target = esv1alpha1.ExternalSecretTarget{ + Name: ExternalSecretTargetSecretName, + Template: &esv1alpha1.ExternalSecretTemplate{ + Type: v1.SecretTypeTLS, + }, + } + + tc.externalSecret.Spec.DataFrom = []esv1alpha1.ExternalSecretDataRemoteRef{ + { + Key: remoteKey, + }, + } + fakeProvider.WithGetSecretMap(map[string][]byte{ + "tls.crt": []byte("map-foo-value"), + "tls.key": []byte("map-bar-value"), + }, nil) + tc.checkSecret = func(es *esv1alpha1.ExternalSecret, secret *v1.Secret) { + Expect(secret.Type).To(Equal(v1.SecretTypeTLS)) + // check values + Expect(string(secret.Data["tls.crt"])).To(Equal("map-foo-value")) + Expect(string(secret.Data["tls.key"])).To(Equal("map-bar-value")) + } + } + // when a provider errors in a GetSecret call // a error condition must be set. providerErrCondition := func(tc *testCase) { @@ -660,6 +798,80 @@ var _ = Describe("ExternalSecret controller", func() { } } + // When the ownership is set to owner, and we delete a dependent child kind=secret + // it should be recreated without waiting for refresh interval + checkDeletion := func(tc *testCase) { + const secretVal = "someValue" + fakeProvider.WithGetSecret([]byte(secretVal), nil) + tc.externalSecret.Spec.RefreshInterval = &metav1.Duration{Duration: time.Minute * 10} + tc.checkSecret = func(es *esv1alpha1.ExternalSecret, secret *v1.Secret) { + + // check values + oldUID := secret.UID + Expect(oldUID).NotTo(BeEmpty()) + + // delete the related config + Expect(k8sClient.Delete(context.TODO(), secret)) + + var newSecret v1.Secret + secretLookupKey := types.NamespacedName{ + Name: ExternalSecretTargetSecretName, + Namespace: ExternalSecretNamespace, + } + Eventually(func() bool { + err := k8sClient.Get(context.Background(), secretLookupKey, &newSecret) + if err != nil { + return false + } + // new secret should be a new, recreated object with a different UID + return newSecret.UID != oldUID + }, timeout, interval).Should(BeTrue()) + } + } + + // Checks that secret annotation has been written based on the data + checkSecretDataHashAnnotation := func(tc *testCase) { + const secretVal = "someValue" + fakeProvider.WithGetSecret([]byte(secretVal), nil) + tc.checkSecret = func(es *esv1alpha1.ExternalSecret, secret *v1.Secret) { + Expect(secret.Annotations[esv1alpha1.AnnotationDataHash]).To(Equal("9d30b95ca81e156f9454b5ef3bfcc6ee")) + } + } + + // When we amend the created kind=secret, refresh operation should be run again regardless of refresh interval + checkSecretDataHashAnnotationChange := func(tc *testCase) { + fakeData := map[string][]byte{ + "targetProperty": []byte("map-foo-value"), + } + fakeProvider.WithGetSecretMap(fakeData, nil) + tc.externalSecret.Spec.RefreshInterval = &metav1.Duration{Duration: time.Minute * 10} + tc.checkSecret = func(es *esv1alpha1.ExternalSecret, secret *v1.Secret) { + oldHash := secret.Annotations[esv1alpha1.AnnotationDataHash] + oldResourceVersion := secret.ResourceVersion + Expect(oldHash).NotTo(BeEmpty()) + + cleanSecret := secret.DeepCopy() + secret.Data["new"] = []byte("value") + secret.ObjectMeta.Annotations[esv1alpha1.AnnotationDataHash] = "thisiswronghash" + Expect(k8sClient.Patch(context.Background(), secret, client.MergeFrom(cleanSecret))).To(Succeed()) + + var refreshedSecret v1.Secret + secretLookupKey := types.NamespacedName{ + Name: ExternalSecretTargetSecretName, + Namespace: ExternalSecretNamespace, + } + Eventually(func() bool { + err := k8sClient.Get(context.Background(), secretLookupKey, &refreshedSecret) + if err != nil { + return false + } + // refreshed secret should have a different generation (sign that it was updated), but since + // the secret source is the same (not changed), the hash should be reverted to an old value + return refreshedSecret.ResourceVersion != oldResourceVersion && refreshedSecret.Annotations[esv1alpha1.AnnotationDataHash] == oldHash + }, timeout, interval).Should(BeTrue()) + } + } + DescribeTable("When reconciling an ExternalSecret", func(tweaks ...testTweaks) { tc := makeDefaultTestcase() @@ -696,16 +908,22 @@ var _ = Describe("ExternalSecret controller", func() { tc.checkSecret(createdES, syncedSecret) } }, + Entry("should recreate deleted secret", checkDeletion), + Entry("should create proper hash annotation for the external secret", checkSecretDataHashAnnotation), + Entry("should refresh when the hash annotation doesn't correspond to secret data", checkSecretDataHashAnnotationChange), Entry("should set the condition eventually", syncLabelsAnnotations), + Entry("should set prometheus counters", checkPrometheusCounters), Entry("should merge with existing secret using creationPolicy=Merge", mergeWithSecret), - Entry("should error if sceret doesn't exist when using creationPolicy=Merge", mergeWithSecretErr), + Entry("should error if secret doesn't exist when using creationPolicy=Merge", mergeWithSecretErr), Entry("should not resolve conflicts with creationPolicy=Merge", mergeWithConflict), Entry("should sync with template", syncWithTemplate), Entry("should sync template with correct value precedence", syncWithTemplatePrecedence), Entry("should refresh secret from template", refreshWithTemplate), + Entry("should be able to use only metadata from template", onlyMetadataFromTemplate), Entry("should refresh secret value when provider secret changes", refreshSecretValue), Entry("should not refresh secret value when provider secret changes but refreshInterval is zero", refreshintervalZero), Entry("should fetch secret using dataFrom", syncWithDataFrom), + Entry("should fetch secret using dataFrom and a template", syncWithDataFromTemplate), Entry("should set error condition when provider errors", providerErrCondition), Entry("should set an error condition when store does not exist", storeMissingErrCondition), Entry("should set an error condition when store provider constructor fails", storeConstructErrCondition), @@ -722,7 +940,6 @@ var _ = Describe("ExternalSecret refresh logic", func() { }, })).To(BeTrue()) }) - It("should refresh when labels change", func() { es := esv1alpha1.ExternalSecret{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/provider/alibaba/fake/fake.go b/pkg/provider/alibaba/fake/fake.go new file mode 100644 index 000000000..6d45239e2 --- /dev/null +++ b/pkg/provider/alibaba/fake/fake.go @@ -0,0 +1,35 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + kmssdk "github.com/aliyun/alibaba-cloud-sdk-go/services/kms" +) + +type AlibabaMockClient struct { + getSecretValue func(request *kmssdk.GetSecretValueRequest) (response *kmssdk.GetSecretValueResponse, err error) +} + +func (mc *AlibabaMockClient) GetSecretValue(*kmssdk.GetSecretValueRequest) (result *kmssdk.GetSecretValueResponse, err error) { + return mc.getSecretValue(&kmssdk.GetSecretValueRequest{}) +} + +func (mc *AlibabaMockClient) WithValue(in *kmssdk.GetSecretValueRequest, val *kmssdk.GetSecretValueResponse, err error) { + if mc != nil { + mc.getSecretValue = func(paramIn *kmssdk.GetSecretValueRequest) (*kmssdk.GetSecretValueResponse, error) { + return val, err + } + } +} diff --git a/pkg/provider/alibaba/kms.go b/pkg/provider/alibaba/kms.go new file mode 100644 index 000000000..c3c3cf223 --- /dev/null +++ b/pkg/provider/alibaba/kms.go @@ -0,0 +1,193 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package alibaba + +import ( + "context" + "encoding/json" + "fmt" + + kmssdk "github.com/aliyun/alibaba-cloud-sdk-go/services/kms" + "github.com/tidwall/gjson" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + kclient "sigs.k8s.io/controller-runtime/pkg/client" + + esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" + "github.com/external-secrets/external-secrets/pkg/provider" + "github.com/external-secrets/external-secrets/pkg/provider/aws/util" + "github.com/external-secrets/external-secrets/pkg/provider/schema" + "github.com/external-secrets/external-secrets/pkg/utils" +) + +const ( + errAlibabaClient = "cannot setup new Alibaba client: %w" + errAlibabaCredSecretName = "invalid Alibaba SecretStore resource: missing Alibaba APIKey" + errUninitalizedAlibabaProvider = "provider Alibaba is not initialized" + errInvalidClusterStoreMissingAKIDNamespace = "invalid ClusterStore, missing AccessKeyID namespace" + errInvalidClusterStoreMissingSKNamespace = "invalid ClusterStore, missing namespace" + errFetchAKIDSecret = "could not fetch AccessKeyID secret: %w" + errMissingSAK = "missing AccessSecretKey" + errMissingAKID = "missing AccessKeyID" +) + +type Client struct { + kube kclient.Client + store *esv1alpha1.AlibabaProvider + namespace string + storeKind string + regionID string + keyID []byte + accessKey []byte +} + +type KeyManagementService struct { + Client SMInterface +} + +type SMInterface interface { + GetSecretValue(request *kmssdk.GetSecretValueRequest) (response *kmssdk.GetSecretValueResponse, err error) +} + +// setAuth creates a new Alibaba session based on a store. +func (c *Client) setAuth(ctx context.Context) error { + credentialsSecret := &corev1.Secret{} + credentialsSecretName := c.store.Auth.SecretRef.AccessKeyID.Name + if credentialsSecretName == "" { + return fmt.Errorf(errAlibabaCredSecretName) + } + objectKey := types.NamespacedName{ + Name: credentialsSecretName, + Namespace: c.namespace, + } + + // only ClusterStore is allowed to set namespace (and then it's required) + if c.storeKind == esv1alpha1.ClusterSecretStoreKind { + if c.store.Auth.SecretRef.AccessKeyID.Namespace == nil { + return fmt.Errorf(errInvalidClusterStoreMissingAKIDNamespace) + } + objectKey.Namespace = *c.store.Auth.SecretRef.AccessKeyID.Namespace + } + + err := c.kube.Get(ctx, objectKey, credentialsSecret) + if err != nil { + return fmt.Errorf(errFetchAKIDSecret, err) + } + + objectKey = types.NamespacedName{ + Name: c.store.Auth.SecretRef.AccessKeySecret.Name, + Namespace: c.namespace, + } + if c.storeKind == esv1alpha1.ClusterSecretStoreKind { + if c.store.Auth.SecretRef.AccessKeySecret.Namespace == nil { + return fmt.Errorf(errInvalidClusterStoreMissingSKNamespace) + } + objectKey.Namespace = *c.store.Auth.SecretRef.AccessKeySecret.Namespace + } + c.keyID = credentialsSecret.Data[c.store.Auth.SecretRef.AccessKeyID.Key] + fmt.Println(c.keyID) + fmt.Println(c.accessKey) + if (c.keyID == nil) || (len(c.keyID) == 0) { + return fmt.Errorf(errMissingAKID) + } + c.accessKey = credentialsSecret.Data[c.store.Auth.SecretRef.AccessKeySecret.Key] + if (c.accessKey == nil) || (len(c.accessKey) == 0) { + return fmt.Errorf(errMissingSAK) + } + c.regionID = c.store.RegionID + return nil +} + +// GetSecret returns a single secret from the provider. +func (kms *KeyManagementService) GetSecret(ctx context.Context, ref esv1alpha1.ExternalSecretDataRemoteRef) ([]byte, error) { + if utils.IsNil(kms.Client) { + return nil, fmt.Errorf(errUninitalizedAlibabaProvider) + } + kmsRequest := kmssdk.CreateGetSecretValueRequest() + kmsRequest.VersionId = ref.Version + kmsRequest.SecretName = ref.Key + kmsRequest.SetScheme("https") + secretOut, err := kms.Client.GetSecretValue(kmsRequest) + if err != nil { + return nil, util.SanitizeErr(err) + } + if ref.Property == "" { + if secretOut.SecretData != "" { + return []byte(secretOut.SecretData), nil + } + return nil, fmt.Errorf("invalid secret received. no secret string nor binary for key: %s", ref.Key) + } + var payload string + if secretOut.SecretData != "" { + payload = secretOut.SecretData + } + val := gjson.Get(payload, ref.Property) + if !val.Exists() { + return nil, fmt.Errorf("key %s does not exist in secret %s", ref.Property, ref.Key) + } + return []byte(val.String()), nil +} + +// GetSecretMap returns multiple k/v pairs from the provider. +func (kms *KeyManagementService) GetSecretMap(ctx context.Context, ref esv1alpha1.ExternalSecretDataRemoteRef) (map[string][]byte, error) { + data, err := kms.GetSecret(ctx, ref) + if err != nil { + return nil, err + } + kv := make(map[string]string) + err = json.Unmarshal(data, &kv) + if err != nil { + return nil, fmt.Errorf("unable to unmarshal secret %s: %w", ref.Key, err) + } + secretData := make(map[string][]byte) + for k, v := range kv { + secretData[k] = []byte(v) + } + return secretData, nil +} + +// NewClient constructs a new secrets client based on the provided store. +func (kms *KeyManagementService) NewClient(ctx context.Context, store esv1alpha1.GenericStore, kube kclient.Client, namespace string) (provider.SecretsClient, error) { + storeSpec := store.GetSpec() + alibabaSpec := storeSpec.Provider.Alibaba + iStore := &Client{ + kube: kube, + store: alibabaSpec, + namespace: namespace, + storeKind: store.GetObjectKind().GroupVersionKind().Kind, + } + if err := iStore.setAuth(ctx); err != nil { + return nil, err + } + alibabaRegion := iStore.regionID + alibabaKeyID := iStore.keyID + alibabaSecretKey := iStore.accessKey + keyManagementService, err := kmssdk.NewClientWithAccessKey(alibabaRegion, string(alibabaKeyID), string(alibabaSecretKey)) + if err != nil { + return nil, fmt.Errorf(errAlibabaClient, err) + } + kms.Client = keyManagementService + return kms, nil +} + +func (kms *KeyManagementService) Close(ctx context.Context) error { + return nil +} + +func init() { + schema.Register(&KeyManagementService{}, &esv1alpha1.SecretStoreProvider{ + Alibaba: &esv1alpha1.AlibabaProvider{}, + }) +} diff --git a/pkg/provider/alibaba/kms_test.go b/pkg/provider/alibaba/kms_test.go new file mode 100644 index 000000000..b69f12b67 --- /dev/null +++ b/pkg/provider/alibaba/kms_test.go @@ -0,0 +1,197 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package alibaba + +import ( + "context" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses" + kmssdk "github.com/aliyun/alibaba-cloud-sdk-go/services/kms" + + esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" + fakesm "github.com/external-secrets/external-secrets/pkg/provider/alibaba/fake" +) + +const ( + secretName = "test-example" + secretValue = "value" +) + +type keyManagementServiceTestCase struct { + mockClient *fakesm.AlibabaMockClient + apiInput *kmssdk.GetSecretValueRequest + apiOutput *kmssdk.GetSecretValueResponse + ref *esv1alpha1.ExternalSecretDataRemoteRef + apiErr error + expectError string + expectedSecret string + // for testing secretmap + expectedData map[string][]byte +} + +func makeValidKMSTestCase() *keyManagementServiceTestCase { + kmstc := keyManagementServiceTestCase{ + mockClient: &fakesm.AlibabaMockClient{}, + apiInput: makeValidAPIInput(), + ref: makeValidRef(), + apiOutput: makeValidAPIOutput(), + apiErr: nil, + expectError: "", + expectedSecret: "", + expectedData: make(map[string][]byte), + } + kmstc.mockClient.WithValue(kmstc.apiInput, kmstc.apiOutput, kmstc.apiErr) + return &kmstc +} + +func makeValidRef() *esv1alpha1.ExternalSecretDataRemoteRef { + return &esv1alpha1.ExternalSecretDataRemoteRef{ + Key: secretName, + } +} + +func makeValidAPIInput() *kmssdk.GetSecretValueRequest { + return &kmssdk.GetSecretValueRequest{ + SecretName: secretName, + } +} + +func makeValidAPIOutput() *kmssdk.GetSecretValueResponse { + kmsresponse := &kmssdk.GetSecretValueResponse{ + BaseResponse: &responses.BaseResponse{}, + RequestId: "", + SecretName: secretName, + VersionId: "", + CreateTime: "", + SecretData: secretValue, + SecretDataType: "", + AutomaticRotation: "", + RotationInterval: "", + NextRotationDate: "", + ExtendedConfig: "", + LastRotationDate: "", + SecretType: "", + VersionStages: kmssdk.VersionStagesInGetSecretValue{}, + } + return kmsresponse +} + +func makeValidKMSTestCaseCustom(tweaks ...func(kmstc *keyManagementServiceTestCase)) *keyManagementServiceTestCase { + kmstc := makeValidKMSTestCase() + for _, fn := range tweaks { + fn(kmstc) + } + kmstc.mockClient.WithValue(kmstc.apiInput, kmstc.apiOutput, kmstc.apiErr) + return kmstc +} + +var setAPIErr = func(kmstc *keyManagementServiceTestCase) { + kmstc.apiErr = fmt.Errorf("oh no") + kmstc.expectError = "oh no" +} + +var setNilMockClient = func(kmstc *keyManagementServiceTestCase) { + kmstc.mockClient = nil + kmstc.expectError = errUninitalizedAlibabaProvider +} + +func TestAlibabaKMSGetSecret(t *testing.T) { + secretData := make(map[string]interface{}) + secretValue := "changedvalue" + secretData["payload"] = secretValue + + // good case: default version is set + // key is passed in, output is sent back + setSecretString := func(kmstc *keyManagementServiceTestCase) { + kmstc.apiOutput.SecretName = secretName + kmstc.apiOutput.SecretData = secretValue + kmstc.expectedSecret = secretValue + } + + // good case: custom version set + setCustomKey := func(kmstc *keyManagementServiceTestCase) { + kmstc.apiOutput.SecretName = "test-example-other" + kmstc.ref.Key = "test-example-other" + kmstc.apiOutput.SecretData = secretValue + kmstc.expectedSecret = secretValue + } + + successCases := []*keyManagementServiceTestCase{ + makeValidKMSTestCaseCustom(setSecretString), + makeValidKMSTestCaseCustom(setCustomKey), + makeValidKMSTestCaseCustom(setAPIErr), + makeValidKMSTestCaseCustom(setNilMockClient), + } + + sm := KeyManagementService{} + for k, v := range successCases { + sm.Client = v.mockClient + out, err := sm.GetSecret(context.Background(), *v.ref) + if !ErrorContains(err, v.expectError) { + t.Errorf("[%d] unexpected error: %s, expected: '%s'", k, err.Error(), v.expectError) + } + if string(out) != v.expectedSecret { + t.Errorf("[%d] unexpected secret: expected %s, got %s", k, v.expectedSecret, string(out)) + } + } +} + +func TestGetSecretMap(t *testing.T) { + // good case: default version & deserialization + setDeserialization := func(kmstc *keyManagementServiceTestCase) { + kmstc.apiOutput.SecretName = "foo" + kmstc.expectedData["foo"] = []byte("bar") + kmstc.apiOutput.SecretData = `{"foo":"bar"}` + } + + // bad case: invalid json + setInvalidJSON := func(kmstc *keyManagementServiceTestCase) { + kmstc.apiOutput.SecretData = "-----------------" + kmstc.expectError = "unable to unmarshal secret" + } + + successCases := []*keyManagementServiceTestCase{ + makeValidKMSTestCaseCustom(setDeserialization), + makeValidKMSTestCaseCustom(setInvalidJSON), + makeValidKMSTestCaseCustom(setNilMockClient), + makeValidKMSTestCaseCustom(setAPIErr), + } + + sm := KeyManagementService{} + for k, v := range successCases { + sm.Client = v.mockClient + out, err := sm.GetSecretMap(context.Background(), *v.ref) + if !ErrorContains(err, v.expectError) { + t.Errorf("[%d] unexpected error: %s, expected: '%s'", k, err.Error(), v.expectError) + } + if err == nil && !reflect.DeepEqual(out, v.expectedData) { + t.Errorf("[%d] unexpected secret data: expected %#v, got %#v", k, v.expectedData, out) + } + } +} + +func ErrorContains(out error, want string) bool { + if out == nil { + return want == "" + } + if want == "" { + return false + } + return strings.Contains(out.Error(), want) +} diff --git a/pkg/provider/aws/parameterstore/parameterstore.go b/pkg/provider/aws/parameterstore/parameterstore.go index 32a18df79..28443149d 100644 --- a/pkg/provider/aws/parameterstore/parameterstore.go +++ b/pkg/provider/aws/parameterstore/parameterstore.go @@ -90,6 +90,6 @@ func (pm *ParameterStore) GetSecretMap(ctx context.Context, ref esv1alpha1.Exter return secretData, nil } -func (pm *ParameterStore) Close() error { +func (pm *ParameterStore) Close(ctx context.Context) error { return nil } diff --git a/pkg/provider/aws/secretsmanager/fake/fake.go b/pkg/provider/aws/secretsmanager/fake/fake.go index 49ce86b84..c8172b5a7 100644 --- a/pkg/provider/aws/secretsmanager/fake/fake.go +++ b/pkg/provider/aws/secretsmanager/fake/fake.go @@ -11,6 +11,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package fake import ( @@ -22,15 +23,38 @@ import ( // Client implements the aws secretsmanager interface. type Client struct { - valFn func(*awssm.GetSecretValueInput) (*awssm.GetSecretValueOutput, error) + ExecutionCounter int + valFn map[string]func(*awssm.GetSecretValueInput) (*awssm.GetSecretValueOutput, error) +} + +// NewClient init a new fake client. +func NewClient() *Client { + return &Client{ + valFn: make(map[string]func(*awssm.GetSecretValueInput) (*awssm.GetSecretValueOutput, error)), + } } func (sm *Client) GetSecretValue(in *awssm.GetSecretValueInput) (*awssm.GetSecretValueOutput, error) { - return sm.valFn(in) + sm.ExecutionCounter++ + if entry, found := sm.valFn[sm.cacheKeyForInput(in)]; found { + return entry(in) + } + return nil, fmt.Errorf("test case not found") +} + +func (sm *Client) cacheKeyForInput(in *awssm.GetSecretValueInput) string { + var secretID, versionID string + if in.SecretId != nil { + secretID = *in.SecretId + } + if in.VersionId != nil { + versionID = *in.VersionId + } + return fmt.Sprintf("%s#%s", secretID, versionID) } func (sm *Client) WithValue(in *awssm.GetSecretValueInput, val *awssm.GetSecretValueOutput, err error) { - sm.valFn = func(paramIn *awssm.GetSecretValueInput) (*awssm.GetSecretValueOutput, error) { + sm.valFn[sm.cacheKeyForInput(in)] = func(paramIn *awssm.GetSecretValueInput) (*awssm.GetSecretValueOutput, error) { if !cmp.Equal(paramIn, in) { return nil, fmt.Errorf("unexpected test argument") } diff --git a/pkg/provider/aws/secretsmanager/secretsmanager.go b/pkg/provider/aws/secretsmanager/secretsmanager.go index 7355345ef..8ddb72c75 100644 --- a/pkg/provider/aws/secretsmanager/secretsmanager.go +++ b/pkg/provider/aws/secretsmanager/secretsmanager.go @@ -11,6 +11,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package secretsmanager import ( @@ -30,6 +31,7 @@ import ( // SecretsManager is a provider for AWS SecretsManager. type SecretsManager struct { client SMInterface + cache map[string]*awssm.GetSecretValueOutput } // SMInterface is a subset of the smiface api. @@ -44,20 +46,37 @@ var log = ctrl.Log.WithName("provider").WithName("aws").WithName("secretsmanager func New(sess client.ConfigProvider) (*SecretsManager, error) { return &SecretsManager{ client: awssm.New(sess), + cache: make(map[string]*awssm.GetSecretValueOutput), }, nil } -// GetSecret returns a single secret from the provider. -func (sm *SecretsManager) GetSecret(ctx context.Context, ref esv1alpha1.ExternalSecretDataRemoteRef) ([]byte, error) { +func (sm *SecretsManager) fetch(_ context.Context, ref esv1alpha1.ExternalSecretDataRemoteRef) (*awssm.GetSecretValueOutput, error) { ver := "AWSCURRENT" if ref.Version != "" { ver = ref.Version } log.Info("fetching secret value", "key", ref.Key, "version", ver) + + cacheKey := fmt.Sprintf("%s#%s", ref.Key, ver) + if secretOut, found := sm.cache[cacheKey]; found { + log.Info("found secret in cache", "key", ref.Key, "version", ver) + return secretOut, nil + } secretOut, err := sm.client.GetSecretValue(&awssm.GetSecretValueInput{ SecretId: &ref.Key, VersionStage: &ver, }) + if err != nil { + return nil, err + } + sm.cache[cacheKey] = secretOut + + return secretOut, nil +} + +// GetSecret returns a single secret from the provider. +func (sm *SecretsManager) GetSecret(ctx context.Context, ref esv1alpha1.ExternalSecretDataRemoteRef) ([]byte, error) { + secretOut, err := sm.fetch(ctx, ref) if err != nil { return nil, util.SanitizeErr(err) } @@ -77,6 +96,7 @@ func (sm *SecretsManager) GetSecret(ctx context.Context, ref esv1alpha1.External if secretOut.SecretBinary != nil { payload = string(secretOut.SecretBinary) } + val := gjson.Get(payload, ref.Property) if !val.Exists() { return nil, fmt.Errorf("key %s does not exist in secret %s", ref.Property, ref.Key) @@ -103,6 +123,6 @@ func (sm *SecretsManager) GetSecretMap(ctx context.Context, ref esv1alpha1.Exter return secretData, nil } -func (sm *SecretsManager) Close() error { +func (sm *SecretsManager) Close(ctx context.Context) error { return nil } diff --git a/pkg/provider/aws/secretsmanager/secretsmanager_test.go b/pkg/provider/aws/secretsmanager/secretsmanager_test.go index dee451b1b..c2aaea714 100644 --- a/pkg/provider/aws/secretsmanager/secretsmanager_test.go +++ b/pkg/provider/aws/secretsmanager/secretsmanager_test.go @@ -11,6 +11,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package secretsmanager import ( @@ -37,11 +38,13 @@ type secretsManagerTestCase struct { expectedSecret string // for testing secretmap expectedData map[string][]byte + // for testing caching + expectedCounter *int } func makeValidSecretsManagerTestCase() *secretsManagerTestCase { smtc := secretsManagerTestCase{ - fakeClient: &fakesm.Client{}, + fakeClient: fakesm.NewClient(), apiInput: makeValidAPIInput(), remoteRef: makeValidRemoteRef(), apiOutput: makeValidAPIOutput(), @@ -164,9 +167,11 @@ func TestSecretsManagerGetSecret(t *testing.T) { makeValidSecretsManagerTestCaseCustom(setAPIErr), } - sm := SecretsManager{} for k, v := range successCases { - sm.client = v.fakeClient + sm := SecretsManager{ + cache: make(map[string]*awssm.GetSecretValueOutput), + client: v.fakeClient, + } out, err := sm.GetSecret(context.Background(), *v.remoteRef) if !ErrorContains(err, v.expectError) { t.Errorf("[%d] unexpected error: %s, expected: '%s'", k, err.Error(), v.expectError) @@ -176,6 +181,58 @@ func TestSecretsManagerGetSecret(t *testing.T) { } } } +func TestCaching(t *testing.T) { + fakeClient := fakesm.NewClient() + + // good case: first call, since we are using the same key, results should be cached and the counter should not go + // over 1 + firstCall := func(smtc *secretsManagerTestCase) { + smtc.apiOutput.SecretString = aws.String(`{"foo":"bar", "bar":"vodka"}`) + smtc.remoteRef.Property = "foo" + smtc.expectedSecret = "bar" + smtc.expectedCounter = aws.Int(1) + smtc.fakeClient = fakeClient + } + secondCall := func(smtc *secretsManagerTestCase) { + smtc.apiOutput.SecretString = aws.String(`{"foo":"bar", "bar":"vodka"}`) + smtc.remoteRef.Property = "bar" + smtc.expectedSecret = "vodka" + smtc.expectedCounter = aws.Int(1) + smtc.fakeClient = fakeClient + } + notCachedCall := func(smtc *secretsManagerTestCase) { + smtc.apiOutput.SecretString = aws.String(`{"sheldon":"bazinga", "bar":"foo"}`) + smtc.remoteRef.Property = "sheldon" + smtc.expectedSecret = "bazinga" + smtc.expectedCounter = aws.Int(2) + smtc.fakeClient = fakeClient + smtc.apiInput.SecretId = aws.String("xyz") + smtc.remoteRef.Key = "xyz" // it should reset the cache since the key is different + } + + cachedCases := []*secretsManagerTestCase{ + makeValidSecretsManagerTestCaseCustom(firstCall), + makeValidSecretsManagerTestCaseCustom(firstCall), + makeValidSecretsManagerTestCaseCustom(secondCall), + makeValidSecretsManagerTestCaseCustom(notCachedCall), + } + sm := SecretsManager{ + cache: make(map[string]*awssm.GetSecretValueOutput), + } + for k, v := range cachedCases { + sm.client = v.fakeClient + out, err := sm.GetSecret(context.Background(), *v.remoteRef) + if !ErrorContains(err, v.expectError) { + t.Errorf("[%d] unexpected error: %s, expected: '%s'", k, err.Error(), v.expectError) + } + if err == nil && string(out) != v.expectedSecret { + t.Errorf("[%d] unexpected secret: expected %s, got %s", k, v.expectedSecret, string(out)) + } + if v.expectedCounter != nil && v.fakeClient.ExecutionCounter != *v.expectedCounter { + t.Errorf("[%d] unexpected counter value: expected %d, got %d", k, v.expectedCounter, v.fakeClient.ExecutionCounter) + } + } +} func TestGetSecretMap(t *testing.T) { // good case: default version & deserialization @@ -184,6 +241,14 @@ func TestGetSecretMap(t *testing.T) { smtc.expectedData["foo"] = []byte("bar") } + // good case: caching + cachedMap := func(smtc *secretsManagerTestCase) { + smtc.apiOutput.SecretString = aws.String(`{"foo":"bar", "plus": "one"}`) + smtc.expectedData["foo"] = []byte("bar") + smtc.expectedData["plus"] = []byte("one") + smtc.expectedCounter = aws.Int(1) + } + // bad case: invalid json setInvalidJSON := func(smtc *secretsManagerTestCase) { smtc.apiOutput.SecretString = aws.String(`-----------------`) @@ -194,11 +259,14 @@ func TestGetSecretMap(t *testing.T) { makeValidSecretsManagerTestCaseCustom(setDeserialization), makeValidSecretsManagerTestCaseCustom(setAPIErr), makeValidSecretsManagerTestCaseCustom(setInvalidJSON), + makeValidSecretsManagerTestCaseCustom(cachedMap), } - sm := SecretsManager{} for k, v := range successCases { - sm.client = v.fakeClient + sm := SecretsManager{ + cache: make(map[string]*awssm.GetSecretValueOutput), + client: v.fakeClient, + } out, err := sm.GetSecretMap(context.Background(), *v.remoteRef) if !ErrorContains(err, v.expectError) { t.Errorf("[%d] unexpected error: %s, expected: '%s'", k, err.Error(), v.expectError) @@ -206,6 +274,9 @@ func TestGetSecretMap(t *testing.T) { if err == nil && !cmp.Equal(out, v.expectedData) { t.Errorf("[%d] unexpected secret data: expected %#v, got %#v", k, v.expectedData, out) } + if v.expectedCounter != nil && v.fakeClient.ExecutionCounter != *v.expectedCounter { + t.Errorf("[%d] unexpected counter value: expected %d, got %d", k, v.expectedCounter, v.fakeClient.ExecutionCounter) + } } } diff --git a/pkg/provider/azure/keyvault/keyvault.go b/pkg/provider/azure/keyvault/keyvault.go index f4358b74d..4a6a0c49a 100644 --- a/pkg/provider/azure/keyvault/keyvault.go +++ b/pkg/provider/azure/keyvault/keyvault.go @@ -227,7 +227,7 @@ func (a *Azure) secretKeyRef(ctx context.Context, namespace string, secretRef sm return value, nil } -func (a *Azure) Close() error { +func (a *Azure) Close(ctx context.Context) error { return nil } diff --git a/pkg/provider/fake/fake.go b/pkg/provider/fake/fake.go index 7854876c3..bbf66fd4b 100644 --- a/pkg/provider/fake/fake.go +++ b/pkg/provider/fake/fake.go @@ -74,7 +74,7 @@ func (v *Client) WithGetSecret(secData []byte, err error) *Client { func (v *Client) GetSecretMap(ctx context.Context, ref esv1alpha1.ExternalSecretDataRemoteRef) (map[string][]byte, error) { return v.GetSecretMapFn(ctx, ref) } -func (v *Client) Close() error { +func (v *Client) Close(ctx context.Context) error { return nil } diff --git a/pkg/provider/gcp/secretmanager/secretsmanager.go b/pkg/provider/gcp/secretmanager/secretsmanager.go index 576b505af..7dd05a065 100644 --- a/pkg/provider/gcp/secretmanager/secretsmanager.go +++ b/pkg/provider/gcp/secretmanager/secretsmanager.go @@ -39,12 +39,12 @@ const ( defaultVersion = "latest" errGCPSMStore = "received invalid GCPSM SecretStore resource" - errGCPSMCredSecretName = "invalid GCPSM SecretStore resource: missing GCP Secret Access Key" errClientClose = "unable to close SecretManager client: %w" errInvalidClusterStoreMissingSAKNamespace = "invalid ClusterSecretStore: missing GCP SecretAccessKey Namespace" errFetchSAKSecret = "could not fetch SecretAccessKey secret: %w" errMissingSAK = "missing SecretAccessKey" errUnableProcessJSONCredentials = "failed to process the provided JSON credentials: %w" + errUnableProcessDefaultCredentials = "failed to process the default credentials: %w" errUnableCreateGCPSMClient = "failed to create GCP secretmanager client: %w" errUninitalizedGCPProvider = "provider GCP is not initialized" errClientGetSecretAccess = "unable to access Secret from SecretManager Client: %w" @@ -73,9 +73,6 @@ type gClient struct { func (c *gClient) setAuth(ctx context.Context) error { credentialsSecret := &corev1.Secret{} credentialsSecretName := c.store.Auth.SecretRef.SecretAccessKey.Name - if credentialsSecretName == "" { - return fmt.Errorf(errGCPSMCredSecretName) - } objectKey := types.NamespacedName{ Name: credentialsSecretName, Namespace: c.namespace, @@ -83,12 +80,16 @@ func (c *gClient) setAuth(ctx context.Context) error { // only ClusterStore is allowed to set namespace (and then it's required) if c.storeKind == esv1alpha1.ClusterSecretStoreKind { - if c.store.Auth.SecretRef.SecretAccessKey.Namespace == nil { + if credentialsSecretName != "" && c.store.Auth.SecretRef.SecretAccessKey.Namespace == nil { return fmt.Errorf(errInvalidClusterStoreMissingSAKNamespace) + } else if credentialsSecretName != "" { + objectKey.Namespace = *c.store.Auth.SecretRef.SecretAccessKey.Namespace } - objectKey.Namespace = *c.store.Auth.SecretRef.SecretAccessKey.Namespace } - + if credentialsSecretName == "" { + c.credentials = nil + return nil + } err := c.kube.Get(ctx, objectKey, credentialsSecret) if err != nil { return fmt.Errorf(errFetchSAKSecret, err) @@ -122,12 +123,23 @@ func (sm *ProviderGCP) NewClient(ctx context.Context, store esv1alpha1.GenericSt sm.projectID = cliStore.store.ProjectID - config, err := google.JWTConfigFromJSON(cliStore.credentials, CloudPlatformRole) - if err != nil { - return nil, fmt.Errorf(errUnableProcessJSONCredentials, err) + if cliStore.credentials != nil { + config, err := google.JWTConfigFromJSON(cliStore.credentials, CloudPlatformRole) + if err != nil { + return nil, fmt.Errorf(errUnableProcessJSONCredentials, err) + } + ts := config.TokenSource(ctx) + clientGCPSM, err := secretmanager.NewClient(ctx, option.WithTokenSource(ts)) + if err != nil { + return nil, fmt.Errorf(errUnableCreateGCPSMClient, err) + } + sm.SecretManagerClient = clientGCPSM + return sm, nil + } + ts, err := google.DefaultTokenSource(ctx, CloudPlatformRole) + if err != nil { + return nil, fmt.Errorf(errUnableProcessDefaultCredentials, err) } - ts := config.TokenSource(ctx) - clientGCPSM, err := secretmanager.NewClient(ctx, option.WithTokenSource(ts)) if err != nil { return nil, fmt.Errorf(errUnableCreateGCPSMClient, err) @@ -199,7 +211,7 @@ func (sm *ProviderGCP) GetSecretMap(ctx context.Context, ref esv1alpha1.External return secretData, nil } -func (sm *ProviderGCP) Close() error { +func (sm *ProviderGCP) Close(ctx context.Context) error { err := sm.SecretManagerClient.Close() if err != nil { return fmt.Errorf(errClientClose, err) diff --git a/pkg/provider/gitlab/fake/fake.go b/pkg/provider/gitlab/fake/fake.go new file mode 100644 index 000000000..d8d116525 --- /dev/null +++ b/pkg/provider/gitlab/fake/fake.go @@ -0,0 +1,39 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package fake + +import ( + gitlab "github.com/xanzy/go-gitlab" +) + +type GitlabMockClient struct { + getVariable func(pid interface{}, key string, options ...gitlab.RequestOptionFunc) (*gitlab.ProjectVariable, *gitlab.Response, error) +} + +func (mc *GitlabMockClient) GetVariable(pid interface{}, key string, options ...gitlab.RequestOptionFunc) (*gitlab.ProjectVariable, *gitlab.Response, error) { + return mc.getVariable(pid, key, nil) +} + +func (mc *GitlabMockClient) WithValue(projectIDinput, keyInput string, output *gitlab.ProjectVariable, err error) { + if mc != nil { + mc.getVariable = func(pid interface{}, key string, options ...gitlab.RequestOptionFunc) (*gitlab.ProjectVariable, *gitlab.Response, error) { + // type secretmanagerpb.AccessSecretVersionRequest contains unexported fields + // use cmpopts.IgnoreUnexported to ignore all the unexported fields in the cmp. + // if !cmp.Equal(paramReq, input, cmpopts.IgnoreUnexported(gitlab.ProjectVariable{})) { + // return nil, nil, fmt.Errorf("unexpected test argument") + // } + return output, nil, err + } + } +} diff --git a/pkg/provider/gitlab/gitlab.go b/pkg/provider/gitlab/gitlab.go new file mode 100644 index 000000000..d59d9e8e0 --- /dev/null +++ b/pkg/provider/gitlab/gitlab.go @@ -0,0 +1,212 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package gitlab + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/tidwall/gjson" + gitlab "github.com/xanzy/go-gitlab" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + kclient "sigs.k8s.io/controller-runtime/pkg/client" + + esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" + "github.com/external-secrets/external-secrets/e2e/framework/log" + "github.com/external-secrets/external-secrets/pkg/provider" + "github.com/external-secrets/external-secrets/pkg/provider/schema" + "github.com/external-secrets/external-secrets/pkg/utils" +) + +// Requires GITLAB_TOKEN and GITLAB_PROJECT_ID to be set in environment variables + +const ( + errGitlabCredSecretName = "credentials are empty" + errInvalidClusterStoreMissingSAKNamespace = "invalid clusterStore missing SAK namespace" + errFetchSAKSecret = "couldn't find secret on cluster: %w" + errMissingSAK = "missing credentials while setting auth" + errUninitalizedGitlabProvider = "provider gitlab is not initialized" + errJSONSecretUnmarshal = "unable to unmarshal secret: %w" +) + +type Client interface { + GetVariable(pid interface{}, key string, options ...gitlab.RequestOptionFunc) (*gitlab.ProjectVariable, *gitlab.Response, error) +} + +// Gitlab Provider struct with reference to a GitLab client and a projectID. +type Gitlab struct { + client Client + projectID interface{} +} + +// Client for interacting with kubernetes cluster...? +type gClient struct { + kube kclient.Client + store *esv1alpha1.GitlabProvider + namespace string + storeKind string + credentials []byte +} + +func init() { + schema.Register(&Gitlab{}, &esv1alpha1.SecretStoreProvider{ + Gitlab: &esv1alpha1.GitlabProvider{}, + }) +} + +// Set gClient credentials to Access Token. +func (c *gClient) setAuth(ctx context.Context) error { + credentialsSecret := &corev1.Secret{} + credentialsSecretName := c.store.Auth.SecretRef.AccessToken.Name + if credentialsSecretName == "" { + return fmt.Errorf(errGitlabCredSecretName) + } + objectKey := types.NamespacedName{ + Name: credentialsSecretName, + Namespace: c.namespace, + } + // only ClusterStore is allowed to set namespace (and then it's required) + if c.storeKind == esv1alpha1.ClusterSecretStoreKind { + if c.store.Auth.SecretRef.AccessToken.Namespace == nil { + return fmt.Errorf(errInvalidClusterStoreMissingSAKNamespace) + } + objectKey.Namespace = *c.store.Auth.SecretRef.AccessToken.Namespace + } + + err := c.kube.Get(ctx, objectKey, credentialsSecret) + if err != nil { + return fmt.Errorf(errFetchSAKSecret, err) + } + + c.credentials = credentialsSecret.Data[c.store.Auth.SecretRef.AccessToken.Key] + if (c.credentials == nil) || (len(c.credentials) == 0) { + return fmt.Errorf(errMissingSAK) + } + // I don't know where ProjectID is being set + // This line SHOULD set it, but instead just breaks everything :) + // c.store.ProjectID = string(credentialsSecret.Data[c.store.ProjectID]) + return nil +} + +// Function newGitlabProvider returns a reference to a new instance of a 'Gitlab' struct. +func NewGitlabProvider() *Gitlab { + return &Gitlab{} +} + +// Method on Gitlab Provider to set up client with credentials and populate projectID. +func (g *Gitlab) NewClient(ctx context.Context, store esv1alpha1.GenericStore, kube kclient.Client, namespace string) (provider.SecretsClient, error) { + storeSpec := store.GetSpec() + if storeSpec == nil || storeSpec.Provider == nil || storeSpec.Provider.Gitlab == nil { + return nil, fmt.Errorf("no store type or wrong store type") + } + storeSpecGitlab := storeSpec.Provider.Gitlab + + cliStore := gClient{ + kube: kube, + store: storeSpecGitlab, + namespace: namespace, + storeKind: store.GetObjectKind().GroupVersionKind().Kind, + } + + if err := cliStore.setAuth(ctx); err != nil { + return nil, err + } + + var err error + + // Create client options + var opts []gitlab.ClientOptionFunc + if cliStore.store.URL != "" { + opts = append(opts, gitlab.WithBaseURL(cliStore.store.URL)) + } + // ClientOptionFunc from the gitlab package can be mapped with the CRD + // in a similar way to extend functionality of the provider + + // Create a new Gitlab client using credentials and options + gitlabClient, err := gitlab.NewClient(string(cliStore.credentials), opts...) + if err != nil { + log.Logf("Failed to create client: %v", err) + } + + g.client = gitlabClient.ProjectVariables + g.projectID = cliStore.store.ProjectID + + return g, nil +} + +func (g *Gitlab) GetSecret(ctx context.Context, ref esv1alpha1.ExternalSecretDataRemoteRef) ([]byte, error) { + if utils.IsNil(g.client) { + return nil, fmt.Errorf(errUninitalizedGitlabProvider) + } + // Need to replace hyphens with underscores to work with Gitlab API + ref.Key = strings.ReplaceAll(ref.Key, "-", "_") + // Retrieves a gitlab variable in the form + // { + // "key": "TEST_VARIABLE_1", + // "variable_type": "env_var", + // "value": "TEST_1", + // "protected": false, + // "masked": true + data, _, err := g.client.GetVariable(g.projectID, ref.Key, nil) // Optional 'filter' parameter could be added later + if err != nil { + return nil, err + } + + if ref.Property == "" { + if data.Value != "" { + return []byte(data.Value), nil + } + return nil, fmt.Errorf("invalid secret received. no secret string for key: %s", ref.Key) + } + + var payload string + if data.Value != "" { + payload = data.Value + } + + val := gjson.Get(payload, ref.Property) + if !val.Exists() { + return nil, fmt.Errorf("key %s does not exist in secret %s", ref.Property, ref.Key) + } + return []byte(val.String()), nil +} + +func (g *Gitlab) GetSecretMap(ctx context.Context, ref esv1alpha1.ExternalSecretDataRemoteRef) (map[string][]byte, error) { + // Gets a secret as normal, expecting secret value to be a json object + data, err := g.GetSecret(ctx, ref) + if err != nil { + return nil, fmt.Errorf("error getting secret %s: %w", ref.Key, err) + } + + // Maps the json data to a string:string map + kv := make(map[string]string) + err = json.Unmarshal(data, &kv) + if err != nil { + return nil, fmt.Errorf(errJSONSecretUnmarshal, err) + } + + // Converts values in K:V pairs into bytes, while leaving keys as strings + secretData := make(map[string][]byte) + for k, v := range kv { + secretData[k] = []byte(v) + } + return secretData, nil +} + +func (g *Gitlab) Close(ctx context.Context) error { + return nil +} diff --git a/pkg/provider/gitlab/gitlab_test.go b/pkg/provider/gitlab/gitlab_test.go new file mode 100644 index 000000000..a9070d027 --- /dev/null +++ b/pkg/provider/gitlab/gitlab_test.go @@ -0,0 +1,178 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package gitlab + +import ( + "context" + "fmt" + "reflect" + "strings" + "testing" + + gitlab "github.com/xanzy/go-gitlab" + + esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" + fakegitlab "github.com/external-secrets/external-secrets/pkg/provider/gitlab/fake" +) + +type secretManagerTestCase struct { + mockClient *fakegitlab.GitlabMockClient + apiInputProjectID string + apiInputKey string + apiOutput *gitlab.ProjectVariable + ref *esv1alpha1.ExternalSecretDataRemoteRef + projectID *string + apiErr error + expectError string + expectedSecret string + // for testing secretmap + expectedData map[string][]byte +} + +func makeValidSecretManagerTestCase() *secretManagerTestCase { + smtc := secretManagerTestCase{ + mockClient: &fakegitlab.GitlabMockClient{}, + apiInputProjectID: makeValidAPIInputProjectID(), + apiInputKey: makeValidAPIInputKey(), + ref: makeValidRef(), + projectID: nil, + apiOutput: makeValidAPIOutput(), + apiErr: nil, + expectError: "", + expectedSecret: "", + expectedData: map[string][]byte{}, + } + smtc.mockClient.WithValue(smtc.apiInputProjectID, smtc.apiInputKey, smtc.apiOutput, smtc.apiErr) + return &smtc +} + +func makeValidRef() *esv1alpha1.ExternalSecretDataRemoteRef { + return &esv1alpha1.ExternalSecretDataRemoteRef{ + Key: "test-secret", + Version: "default", + } +} + +func makeValidAPIInputProjectID() string { + return "testID" +} + +func makeValidAPIInputKey() string { + return "testKey" +} + +func makeValidAPIOutput() *gitlab.ProjectVariable { + return &gitlab.ProjectVariable{ + Key: "testKey", + Value: "", + } +} + +func makeValidSecretManagerTestCaseCustom(tweaks ...func(smtc *secretManagerTestCase)) *secretManagerTestCase { + smtc := makeValidSecretManagerTestCase() + for _, fn := range tweaks { + fn(smtc) + } + smtc.mockClient.WithValue(smtc.apiInputProjectID, smtc.apiInputKey, smtc.apiOutput, smtc.apiErr) + return smtc +} + +// This case can be shared by both GetSecret and GetSecretMap tests. +// bad case: set apiErr. +var setAPIErr = func(smtc *secretManagerTestCase) { + smtc.apiErr = fmt.Errorf("oh no") + smtc.expectError = "oh no" +} + +var setNilMockClient = func(smtc *secretManagerTestCase) { + smtc.mockClient = nil + smtc.expectError = errUninitalizedGitlabProvider +} + +// test the sm<->gcp interface +// make sure correct values are passed and errors are handled accordingly. +func TestGitlabSecretManagerGetSecret(t *testing.T) { + secretValue := "changedvalue" + // good case: default version is set + // key is passed in, output is sent back + + setSecretString := func(smtc *secretManagerTestCase) { + smtc.apiOutput = &gitlab.ProjectVariable{ + Key: "testkey", + Value: "changedvalue", + } + smtc.expectedSecret = secretValue + } + + successCases := []*secretManagerTestCase{ + makeValidSecretManagerTestCaseCustom(setSecretString), + makeValidSecretManagerTestCaseCustom(setAPIErr), + makeValidSecretManagerTestCaseCustom(setNilMockClient), + } + + sm := Gitlab{} + for k, v := range successCases { + sm.client = v.mockClient + out, err := sm.GetSecret(context.Background(), *v.ref) + if !ErrorContains(err, v.expectError) { + t.Errorf("[%d] unexpected error: %s, expected: '%s'", k, err.Error(), v.expectError) + } + if string(out) != v.expectedSecret { + t.Errorf("[%d] unexpected secret: expected %s, got %s", k, v.expectedSecret, string(out)) + } + } +} + +func TestGetSecretMap(t *testing.T) { + // good case: default version & deserialization + setDeserialization := func(smtc *secretManagerTestCase) { + smtc.apiOutput.Value = `{"foo":"bar"}` + smtc.expectedData["foo"] = []byte("bar") + } + + // bad case: invalid json + setInvalidJSON := func(smtc *secretManagerTestCase) { + smtc.apiOutput.Value = `-----------------` + smtc.expectError = "unable to unmarshal secret" + } + + successCases := []*secretManagerTestCase{ + makeValidSecretManagerTestCaseCustom(setDeserialization), + makeValidSecretManagerTestCaseCustom(setInvalidJSON), + makeValidSecretManagerTestCaseCustom(setNilMockClient), + makeValidSecretManagerTestCaseCustom(setAPIErr), + } + + sm := Gitlab{} + for k, v := range successCases { + sm.client = v.mockClient + out, err := sm.GetSecretMap(context.Background(), *v.ref) + if !ErrorContains(err, v.expectError) { + t.Errorf("[%d] unexpected error: %s, expected: '%s'", k, err.Error(), v.expectError) + } + if err == nil && !reflect.DeepEqual(out, v.expectedData) { + t.Errorf("[%d] unexpected secret data: expected %#v, got %#v", k, v.expectedData, out) + } + } +} + +func ErrorContains(out error, want string) bool { + if out == nil { + return want == "" + } + if want == "" { + return false + } + return strings.Contains(out.Error(), want) +} diff --git a/pkg/provider/ibm/provider.go b/pkg/provider/ibm/provider.go index 14aeec052..fa64e6c55 100644 --- a/pkg/provider/ibm/provider.go +++ b/pkg/provider/ibm/provider.go @@ -289,7 +289,7 @@ func (ibm *providerIBM) GetSecretMap(ctx context.Context, ref esv1alpha1.Externa } } -func (ibm *providerIBM) Close() error { +func (ibm *providerIBM) Close(ctx context.Context) error { return nil } diff --git a/pkg/provider/oracle/fake/fake.go b/pkg/provider/oracle/fake/fake.go new file mode 100644 index 000000000..f5a2bd41c --- /dev/null +++ b/pkg/provider/oracle/fake/fake.go @@ -0,0 +1,36 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package fake + +import ( + "context" + + vault "github.com/oracle/oci-go-sdk/v45/vault" +) + +type OracleMockClient struct { + getSecret func(ctx context.Context, request vault.GetSecretRequest) (response vault.GetSecretResponse, err error) +} + +func (mc *OracleMockClient) GetSecret(ctx context.Context, request vault.GetSecretRequest) (response vault.GetSecretResponse, err error) { + return mc.getSecret(ctx, request) +} + +func (mc *OracleMockClient) WithValue(input vault.GetSecretRequest, output vault.GetSecretResponse, err error) { + if mc != nil { + mc.getSecret = func(ctx context.Context, paramReq vault.GetSecretRequest) (vault.GetSecretResponse, error) { + return output, err + } + } +} diff --git a/pkg/provider/oracle/oracle.go b/pkg/provider/oracle/oracle.go new file mode 100644 index 000000000..4ac27fe1c --- /dev/null +++ b/pkg/provider/oracle/oracle.go @@ -0,0 +1,213 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package oracle + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/oracle/oci-go-sdk/v45/common" + vault "github.com/oracle/oci-go-sdk/v45/vault" + "github.com/tidwall/gjson" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + kclient "sigs.k8s.io/controller-runtime/pkg/client" + + esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" + "github.com/external-secrets/external-secrets/pkg/provider" + "github.com/external-secrets/external-secrets/pkg/provider/aws/util" + "github.com/external-secrets/external-secrets/pkg/provider/schema" + "github.com/external-secrets/external-secrets/pkg/utils" +) + +const ( + VaultEndpointEnv = "ORACLE_VAULT_ENDPOINT" + STSEndpointEnv = "ORACLE_STS_ENDPOINT" + SVMEndpointEnv = "ORACLE_SVM_ENDPOINT" + + errOracleClient = "cannot setup new oracle client: %w" + errORACLECredSecretName = "invalid oracle SecretStore resource: missing oracle APIKey" + errUninitalizedOracleProvider = "provider oracle is not initialized" + errInvalidClusterStoreMissingSKNamespace = "invalid ClusterStore, missing namespace" + errFetchSAKSecret = "could not fetch SecretAccessKey secret: %w" + errMissingPK = "missing PrivateKey" + errMissingUser = "missing User ID" + errMissingTenancy = "missing Tenancy ID" + errMissingRegion = "missing Region" + errMissingFingerprint = "missing Fingerprint" + errJSONSecretUnmarshal = "unable to unmarshal secret: %w" + errMissingKey = "missing Key in secret: %s" + errInvalidSecret = "invalid secret received. no secret string nor binary for key: %s" +) + +type client struct { + kube kclient.Client + store *esv1alpha1.OracleProvider + namespace string + storeKind string + tenancy string + user string + region string + fingerprint string + privateKey string +} + +type VaultManagementService struct { + Client VMInterface +} + +type VMInterface interface { + GetSecret(ctx context.Context, request vault.GetSecretRequest) (response vault.GetSecretResponse, err error) +} + +func (c *client) setAuth(ctx context.Context) error { + credentialsSecret := &corev1.Secret{} + credentialsSecretName := c.store.Auth.SecretRef.PrivateKey.Name + if credentialsSecretName == "" { + return fmt.Errorf(errORACLECredSecretName) + } + objectKey := types.NamespacedName{ + Name: credentialsSecretName, + Namespace: c.namespace, + } + + // only ClusterStore is allowed to set namespace (and then it's required) + if c.storeKind == esv1alpha1.ClusterSecretStoreKind { + if c.store.Auth.SecretRef.PrivateKey.Namespace == nil { + return fmt.Errorf(errInvalidClusterStoreMissingSKNamespace) + } + objectKey.Namespace = *c.store.Auth.SecretRef.PrivateKey.Namespace + } + + err := c.kube.Get(ctx, objectKey, credentialsSecret) + if err != nil { + return fmt.Errorf(errFetchSAKSecret, err) + } + + c.privateKey = string(credentialsSecret.Data[c.store.Auth.SecretRef.PrivateKey.Key]) + if c.privateKey == "" { + return fmt.Errorf(errMissingPK) + } + + c.fingerprint = string(credentialsSecret.Data[c.store.Auth.SecretRef.Fingerprint.Key]) + if c.fingerprint == "" { + return fmt.Errorf(errMissingFingerprint) + } + + c.user = c.store.User + if c.user == "" { + return fmt.Errorf(errMissingUser) + } + + c.tenancy = c.store.Tenancy + if c.tenancy == "" { + return fmt.Errorf(errMissingTenancy) + } + + c.region = c.store.Region + if c.region == "" { + return fmt.Errorf(errMissingRegion) + } + + return nil +} + +func (vms *VaultManagementService) GetSecret(ctx context.Context, ref esv1alpha1.ExternalSecretDataRemoteRef) ([]byte, error) { + if utils.IsNil(vms.Client) { + return nil, fmt.Errorf(errUninitalizedOracleProvider) + } + vmsRequest := vault.GetSecretRequest{ + SecretId: &ref.Key, + } + secretOut, err := vms.Client.GetSecret(context.Background(), vmsRequest) + if err != nil { + return nil, util.SanitizeErr(err) + } + if ref.Property == "" { + if *secretOut.SecretName != "" { + return []byte(*secretOut.SecretName), nil + } + return nil, fmt.Errorf(errInvalidSecret, ref.Key) + } + var payload *string + if secretOut.SecretName != nil { + payload = secretOut.SecretName + } + + payloadval := *payload + + val := gjson.Get(payloadval, ref.Property) + if !val.Exists() { + return nil, fmt.Errorf(errMissingKey, ref.Key) + } + + return []byte(val.String()), nil +} + +func (vms *VaultManagementService) GetSecretMap(ctx context.Context, ref esv1alpha1.ExternalSecretDataRemoteRef) (map[string][]byte, error) { + data, err := vms.GetSecret(ctx, ref) + if err != nil { + return nil, err + } + kv := make(map[string]string) + err = json.Unmarshal(data, &kv) + if err != nil { + return nil, fmt.Errorf(errJSONSecretUnmarshal, err) + } + secretData := make(map[string][]byte) + for k, v := range kv { + secretData[k] = []byte(v) + } + return secretData, nil +} + +// NewClient constructs a new secrets client based on the provided store. +func (vms *VaultManagementService) NewClient(ctx context.Context, store esv1alpha1.GenericStore, kube kclient.Client, namespace string) (provider.SecretsClient, error) { + storeSpec := store.GetSpec() + oracleSpec := storeSpec.Provider.Oracle + + oracleStore := &client{ + kube: kube, + store: oracleSpec, + namespace: namespace, + storeKind: store.GetObjectKind().GroupVersionKind().Kind, + } + if err := oracleStore.setAuth(ctx); err != nil { + return nil, err + } + + oracleTenancy := oracleStore.tenancy + oracleUser := oracleStore.user + oracleRegion := oracleStore.region + oracleFingerprint := oracleStore.fingerprint + oraclePrivateKey := oracleStore.privateKey + + configurationProvider := common.NewRawConfigurationProvider(oracleTenancy, oracleUser, oracleRegion, oracleFingerprint, oraclePrivateKey, nil) + + vaultManagementService, err := vault.NewVaultsClientWithConfigurationProvider(configurationProvider) + if err != nil { + return nil, fmt.Errorf(errOracleClient, err) + } + vms.Client = vaultManagementService + return vms, nil +} + +func (vms *VaultManagementService) Close(ctx context.Context) error { + return nil +} + +func init() { + schema.Register(&VaultManagementService{}, &esv1alpha1.SecretStoreProvider{ + Oracle: &esv1alpha1.OracleProvider{}, + }) +} diff --git a/pkg/provider/oracle/oracle_test.go b/pkg/provider/oracle/oracle_test.go new file mode 100644 index 000000000..95ec2ce55 --- /dev/null +++ b/pkg/provider/oracle/oracle_test.go @@ -0,0 +1,173 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package oracle + +import ( + "context" + "fmt" + "reflect" + "strings" + "testing" + + vault "github.com/oracle/oci-go-sdk/v45/vault" + utilpointer "k8s.io/utils/pointer" + + esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" + fakeoracle "github.com/external-secrets/external-secrets/pkg/provider/oracle/fake" +) + +type vaultTestCase struct { + mockClient *fakeoracle.OracleMockClient + apiInput *vault.GetSecretRequest + apiOutput *vault.GetSecretResponse + ref *esv1alpha1.ExternalSecretDataRemoteRef + apiErr error + expectError string + expectedSecret string + // for testing secretmap + expectedData map[string][]byte +} + +func makeValidVaultTestCase() *vaultTestCase { + smtc := vaultTestCase{ + mockClient: &fakeoracle.OracleMockClient{}, + apiInput: makeValidAPIInput(), + ref: makeValidRef(), + apiOutput: makeValidAPIOutput(), + apiErr: nil, + expectError: "", + expectedSecret: "", + expectedData: map[string][]byte{}, + } + smtc.mockClient.WithValue(*smtc.apiInput, *smtc.apiOutput, smtc.apiErr) + return &smtc +} + +func makeValidRef() *esv1alpha1.ExternalSecretDataRemoteRef { + return &esv1alpha1.ExternalSecretDataRemoteRef{ + Key: "test-secret", + Version: "default", + } +} + +func makeValidAPIInput() *vault.GetSecretRequest { + return &vault.GetSecretRequest{ + SecretId: utilpointer.StringPtr("test-secret"), + } +} + +func makeValidAPIOutput() *vault.GetSecretResponse { + return &vault.GetSecretResponse{ + Etag: utilpointer.StringPtr("test-name"), + Secret: vault.Secret{}, + } +} + +func makeValidVaultTestCaseCustom(tweaks ...func(smtc *vaultTestCase)) *vaultTestCase { + smtc := makeValidVaultTestCase() + for _, fn := range tweaks { + fn(smtc) + } + smtc.mockClient.WithValue(*smtc.apiInput, *smtc.apiOutput, smtc.apiErr) + return smtc +} + +// This case can be shared by both GetSecret and GetSecretMap tests. +// bad case: set apiErr. +var setAPIErr = func(smtc *vaultTestCase) { + smtc.apiErr = fmt.Errorf("oh no") + smtc.expectError = "oh no" +} + +var setNilMockClient = func(smtc *vaultTestCase) { + smtc.mockClient = nil + smtc.expectError = errUninitalizedOracleProvider +} + +func TestOracleVaultGetSecret(t *testing.T) { + secretValue := "changedvalue" + // good case: default version is set + // key is passed in, output is sent back + setSecretString := func(smtc *vaultTestCase) { + smtc.apiOutput = &vault.GetSecretResponse{ + Etag: utilpointer.StringPtr("test-name"), + Secret: vault.Secret{ + CompartmentId: utilpointer.StringPtr("test-compartment-id"), + Id: utilpointer.StringPtr("test-id"), + SecretName: utilpointer.StringPtr("changedvalue"), + }, + } + smtc.expectedSecret = secretValue + } + + successCases := []*vaultTestCase{ + makeValidVaultTestCaseCustom(setAPIErr), + makeValidVaultTestCaseCustom(setNilMockClient), + makeValidVaultTestCaseCustom(setSecretString), + } + + sm := VaultManagementService{} + for k, v := range successCases { + sm.Client = v.mockClient + fmt.Println(*v.ref) + out, err := sm.GetSecret(context.Background(), *v.ref) + if !ErrorContains(err, v.expectError) { + t.Errorf("[%d] unexpected error: %s, expected: '%s'", k, err.Error(), v.expectError) + } + if string(out) != v.expectedSecret { + t.Errorf("[%d] unexpected secret: expected %s, got %s", k, v.expectedSecret, string(out)) + } + } +} + +func TestGetSecretMap(t *testing.T) { + // good case: default version & deserialization + setDeserialization := func(smtc *vaultTestCase) { + smtc.apiOutput.SecretName = utilpointer.StringPtr(`{"foo":"bar"}`) + smtc.expectedData["foo"] = []byte("bar") + } + + // bad case: invalid json + setInvalidJSON := func(smtc *vaultTestCase) { + smtc.apiOutput.SecretName = utilpointer.StringPtr(`-----------------`) + smtc.expectError = "unable to unmarshal secret" + } + + successCases := []*vaultTestCase{ + makeValidVaultTestCaseCustom(setDeserialization), + makeValidVaultTestCaseCustom(setInvalidJSON), + makeValidVaultTestCaseCustom(setNilMockClient), + makeValidVaultTestCaseCustom(setAPIErr), + } + + sm := VaultManagementService{} + for k, v := range successCases { + sm.Client = v.mockClient + out, err := sm.GetSecretMap(context.Background(), *v.ref) + if !ErrorContains(err, v.expectError) { + t.Errorf("[%d] unexpected error: %s, expected: '%s'", k, err.Error(), v.expectError) + } + if err == nil && !reflect.DeepEqual(out, v.expectedData) { + t.Errorf("[%d] unexpected secret data: expected %#v, got %#v", k, v.expectedData, out) + } + } +} + +func ErrorContains(out error, want string) bool { + if out == nil { + return want == "" + } + if want == "" { + return false + } + return strings.Contains(out.Error(), want) +} diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 55254f243..0a3ac201a 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -35,5 +35,5 @@ type SecretsClient interface { // GetSecretMap returns multiple k/v pairs from the provider GetSecretMap(ctx context.Context, ref esv1alpha1.ExternalSecretDataRemoteRef) (map[string][]byte, error) - Close() error + Close(ctx context.Context) error } diff --git a/pkg/provider/register/register.go b/pkg/provider/register/register.go index c71274795..8a0a208aa 100644 --- a/pkg/provider/register/register.go +++ b/pkg/provider/register/register.go @@ -17,9 +17,13 @@ package register // packages imported here are registered to the controller schema. // nolint:golint import ( + _ "github.com/external-secrets/external-secrets/pkg/provider/alibaba" _ "github.com/external-secrets/external-secrets/pkg/provider/aws" _ "github.com/external-secrets/external-secrets/pkg/provider/azure/keyvault" _ "github.com/external-secrets/external-secrets/pkg/provider/gcp/secretmanager" + _ "github.com/external-secrets/external-secrets/pkg/provider/gitlab" _ "github.com/external-secrets/external-secrets/pkg/provider/ibm" + _ "github.com/external-secrets/external-secrets/pkg/provider/oracle" _ "github.com/external-secrets/external-secrets/pkg/provider/vault" + _ "github.com/external-secrets/external-secrets/pkg/provider/yandex/lockbox" ) diff --git a/pkg/provider/schema/schema.go b/pkg/provider/schema/schema.go index 82e5e308c..9c2a0d626 100644 --- a/pkg/provider/schema/schema.go +++ b/pkg/provider/schema/schema.go @@ -92,7 +92,7 @@ func GetProvider(s esv1alpha1.GenericStore) (provider.Provider, error) { // or an error if the provider is not configured. func getProviderName(storeSpec *esv1alpha1.SecretStoreProvider) (string, error) { storeBytes, err := json.Marshal(storeSpec) - if err != nil { + if err != nil || storeBytes == nil { return "", fmt.Errorf("failed to marshal store spec: %w", err) } diff --git a/pkg/provider/schema/schema_test.go b/pkg/provider/schema/schema_test.go index 588cc7746..c3023fb14 100644 --- a/pkg/provider/schema/schema_test.go +++ b/pkg/provider/schema/schema_test.go @@ -41,7 +41,7 @@ func (p *PP) GetSecretMap(ctx context.Context, ref esv1alpha1.ExternalSecretData return map[string][]byte{}, nil } -func (p *PP) Close() error { +func (p *PP) Close(ctx context.Context) error { return nil } diff --git a/pkg/provider/vault/vault.go b/pkg/provider/vault/vault.go index 398b78f38..ee6ab7dbf 100644 --- a/pkg/provider/vault/vault.go +++ b/pkg/provider/vault/vault.go @@ -155,7 +155,7 @@ func (v *client) GetSecretMap(ctx context.Context, ref esv1alpha1.ExternalSecret return v.readSecret(ctx, ref.Key, ref.Version) } -func (v *client) Close() error { +func (v *client) Close(ctx context.Context) error { return nil } diff --git a/pkg/provider/yandex/lockbox/client/client.go b/pkg/provider/yandex/lockbox/client/client.go new file mode 100644 index 000000000..a1739b286 --- /dev/null +++ b/pkg/provider/yandex/lockbox/client/client.go @@ -0,0 +1,39 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package client + +import ( + "context" + "time" + + "github.com/yandex-cloud/go-genproto/yandex/cloud/lockbox/v1" + "github.com/yandex-cloud/go-sdk/iamkey" +) + +// Creates Lockbox clients and Yandex.Cloud IAM tokens. +type YandexCloudCreator interface { + CreateLockboxClient(ctx context.Context, apiEndpoint string, authorizedKey *iamkey.Key) (LockboxClient, error) + CreateIamToken(ctx context.Context, apiEndpoint string, authorizedKey *iamkey.Key) (*IamToken, error) + Now() time.Time +} + +type IamToken struct { + Token string + ExpiresAt time.Time +} + +// Responsible for accessing Lockbox secrets. +type LockboxClient interface { + GetPayloadEntries(ctx context.Context, iamToken, secretID, versionID string) ([]*lockbox.Payload_Entry, error) +} diff --git a/pkg/provider/yandex/lockbox/client/fake/fake.go b/pkg/provider/yandex/lockbox/client/fake/fake.go new file mode 100644 index 000000000..891c5bd9e --- /dev/null +++ b/pkg/provider/yandex/lockbox/client/fake/fake.go @@ -0,0 +1,151 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package fake + +import ( + "context" + "fmt" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "github.com/yandex-cloud/go-genproto/yandex/cloud/lockbox/v1" + "github.com/yandex-cloud/go-sdk/iamkey" + + "github.com/external-secrets/external-secrets/pkg/provider/yandex/lockbox/client" +) + +// Fake implementation of YandexCloudCreator. +type YandexCloudCreator struct { + Backend *LockboxBackend +} + +func (c *YandexCloudCreator) CreateLockboxClient(ctx context.Context, apiEndpoint string, authorizedKey *iamkey.Key) (client.LockboxClient, error) { + return &LockboxClient{c.Backend}, nil +} + +func (c *YandexCloudCreator) CreateIamToken(ctx context.Context, apiEndpoint string, authorizedKey *iamkey.Key) (*client.IamToken, error) { + return c.Backend.getToken(authorizedKey) +} + +func (c *YandexCloudCreator) Now() time.Time { + return c.Backend.now +} + +// Fake implementation of LockboxClient. +type LockboxClient struct { + fakeLockboxBackend *LockboxBackend +} + +func (c *LockboxClient) GetPayloadEntries(ctx context.Context, iamToken, secretID, versionID string) ([]*lockbox.Payload_Entry, error) { + return c.fakeLockboxBackend.getEntries(iamToken, secretID, versionID) +} + +// Fakes Yandex Lockbox service backend. +type LockboxBackend struct { + secretMap map[secretKey]secretValue // secret specific data + versionMap map[versionKey]versionValue // version specific data + tokenMap map[tokenKey]tokenValue // token specific data + + tokenExpirationDuration time.Duration + now time.Time // fakes the current time +} + +type secretKey struct { + secretID string +} + +type secretValue struct { + expectedAuthorizedKey *iamkey.Key // authorized key expected to access the secret +} + +type versionKey struct { + secretID string + versionID string +} + +type versionValue struct { + entries []*lockbox.Payload_Entry +} + +type tokenKey struct { + token string +} + +type tokenValue struct { + authorizedKey *iamkey.Key + expiresAt time.Time +} + +func NewLockboxBackend(tokenExpirationDuration time.Duration) *LockboxBackend { + return &LockboxBackend{ + secretMap: make(map[secretKey]secretValue), + versionMap: make(map[versionKey]versionValue), + tokenMap: make(map[tokenKey]tokenValue), + tokenExpirationDuration: tokenExpirationDuration, + now: time.Time{}, + } +} + +func (lb *LockboxBackend) CreateSecret(authorizedKey *iamkey.Key, entries ...*lockbox.Payload_Entry) (string, string) { + secretID := uuid.NewString() + versionID := uuid.NewString() + + lb.secretMap[secretKey{secretID}] = secretValue{authorizedKey} + lb.versionMap[versionKey{secretID, ""}] = versionValue{entries} // empty versionID corresponds to the latest version + lb.versionMap[versionKey{secretID, versionID}] = versionValue{entries} + + return secretID, versionID +} + +func (lb *LockboxBackend) AddVersion(secretID string, entries ...*lockbox.Payload_Entry) string { + versionID := uuid.NewString() + + lb.versionMap[versionKey{secretID, ""}] = versionValue{entries} // empty versionID corresponds to the latest version + lb.versionMap[versionKey{secretID, versionID}] = versionValue{entries} + + return versionID +} + +func (lb *LockboxBackend) AdvanceClock(duration time.Duration) { + lb.now = lb.now.Add(duration) +} + +func (lb *LockboxBackend) getToken(authorizedKey *iamkey.Key) (*client.IamToken, error) { + token := uuid.NewString() + expiresAt := lb.now.Add(lb.tokenExpirationDuration) + lb.tokenMap[tokenKey{token}] = tokenValue{authorizedKey, expiresAt} + return &client.IamToken{Token: token, ExpiresAt: expiresAt}, nil +} + +func (lb *LockboxBackend) getEntries(iamToken, secretID, versionID string) ([]*lockbox.Payload_Entry, error) { + if _, ok := lb.secretMap[secretKey{secretID}]; !ok { + return nil, fmt.Errorf("secret not found") + } + if _, ok := lb.versionMap[versionKey{secretID, versionID}]; !ok { + return nil, fmt.Errorf("version not found") + } + if _, ok := lb.tokenMap[tokenKey{iamToken}]; !ok { + return nil, fmt.Errorf("unauthenticated") + } + + if lb.tokenMap[tokenKey{iamToken}].expiresAt.Before(lb.now) { + return nil, fmt.Errorf("iam token expired") + } + if !cmp.Equal(lb.tokenMap[tokenKey{iamToken}].authorizedKey, lb.secretMap[secretKey{secretID}].expectedAuthorizedKey) { + return nil, fmt.Errorf("permission denied") + } + + return lb.versionMap[versionKey{secretID, versionID}].entries, nil +} diff --git a/pkg/provider/yandex/lockbox/client/grpc/grpc.go b/pkg/provider/yandex/lockbox/client/grpc/grpc.go new file mode 100644 index 000000000..898ebf9a2 --- /dev/null +++ b/pkg/provider/yandex/lockbox/client/grpc/grpc.go @@ -0,0 +1,144 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package grpc + +import ( + "context" + "crypto/tls" + "time" + + "github.com/yandex-cloud/go-genproto/yandex/cloud/endpoint" + "github.com/yandex-cloud/go-genproto/yandex/cloud/lockbox/v1" + ycsdk "github.com/yandex-cloud/go-sdk" + "github.com/yandex-cloud/go-sdk/iamkey" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/keepalive" + + "github.com/external-secrets/external-secrets/pkg/provider/yandex/lockbox/client" +) + +// Implementation of YandexCloudCreator. +type YandexCloudCreator struct { +} + +func (lb *YandexCloudCreator) CreateLockboxClient(ctx context.Context, apiEndpoint string, authorizedKey *iamkey.Key) (client.LockboxClient, error) { + sdk, err := buildSDK(ctx, apiEndpoint, authorizedKey) + if err != nil { + return nil, err + } + + payloadAPIEndpoint, err := sdk.ApiEndpoint().ApiEndpoint().Get(ctx, &endpoint.GetApiEndpointRequest{ + ApiEndpointId: "lockbox-payload", // the ID from https://api.cloud.yandex.net/endpoints + }) + if err != nil { + return nil, err + } + + err = closeSDK(ctx, sdk) + if err != nil { + return nil, err + } + + conn, err := grpc.Dial(payloadAPIEndpoint.Address, + grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS12})), + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: time.Second * 30, + Timeout: time.Second * 10, + PermitWithoutStream: false, + }), + grpc.WithUserAgent("external-secrets"), + ) + if err != nil { + return nil, err + } + + return &LockboxClient{lockbox.NewPayloadServiceClient(conn)}, nil +} + +func (lb *YandexCloudCreator) CreateIamToken(ctx context.Context, apiEndpoint string, authorizedKey *iamkey.Key) (*client.IamToken, error) { + sdk, err := buildSDK(ctx, apiEndpoint, authorizedKey) + if err != nil { + return nil, err + } + + iamToken, err := sdk.CreateIAMToken(ctx) + if err != nil { + return nil, err + } + + err = closeSDK(ctx, sdk) + if err != nil { + return nil, err + } + + return &client.IamToken{Token: iamToken.IamToken, ExpiresAt: iamToken.ExpiresAt.AsTime()}, nil +} + +func (lb *YandexCloudCreator) Now() time.Time { + return time.Now() +} + +func buildSDK(ctx context.Context, apiEndpoint string, authorizedKey *iamkey.Key) (*ycsdk.SDK, error) { + creds, err := ycsdk.ServiceAccountKey(authorizedKey) + if err != nil { + return nil, err + } + + sdk, err := ycsdk.Build(ctx, ycsdk.Config{ + Credentials: creds, + Endpoint: apiEndpoint, + }) + if err != nil { + return nil, err + } + + return sdk, nil +} + +func closeSDK(ctx context.Context, sdk *ycsdk.SDK) error { + return sdk.Shutdown(ctx) +} + +// Implementation of LockboxClient. +type LockboxClient struct { + lockboxPayloadClient lockbox.PayloadServiceClient +} + +func (lc *LockboxClient) GetPayloadEntries(ctx context.Context, iamToken, secretID, versionID string) ([]*lockbox.Payload_Entry, error) { + payload, err := lc.lockboxPayloadClient.Get( + ctx, + &lockbox.GetPayloadRequest{ + SecretId: secretID, + VersionId: versionID, + }, + grpc.PerRPCCredentials(perRPCCredentials{iamToken: iamToken}), + ) + if err != nil { + return nil, err + } + return payload.Entries, nil +} + +type perRPCCredentials struct { + iamToken string +} + +func (t perRPCCredentials) GetRequestMetadata(ctx context.Context, in ...string) (map[string]string, error) { + return map[string]string{"Authorization": "Bearer " + t.iamToken}, nil +} + +func (perRPCCredentials) RequireTransportSecurity() bool { + return true +} diff --git a/pkg/provider/yandex/lockbox/lockbox.go b/pkg/provider/yandex/lockbox/lockbox.go new file mode 100644 index 000000000..f096c2dbf --- /dev/null +++ b/pkg/provider/yandex/lockbox/lockbox.go @@ -0,0 +1,299 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package lockbox + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "sync" + "time" + + "github.com/yandex-cloud/go-genproto/yandex/cloud/lockbox/v1" + "github.com/yandex-cloud/go-sdk/iamkey" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + kclient "sigs.k8s.io/controller-runtime/pkg/client" + + esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" + "github.com/external-secrets/external-secrets/pkg/provider" + "github.com/external-secrets/external-secrets/pkg/provider/schema" + "github.com/external-secrets/external-secrets/pkg/provider/yandex/lockbox/client" + "github.com/external-secrets/external-secrets/pkg/provider/yandex/lockbox/client/grpc" +) + +const maxSecretsClientLifetime = 5 * time.Minute // supposed SecretsClient lifetime is quite short +const iamTokenCleanupDelay = 1 * time.Hour // specifies how often cleanUpIamTokenMap() is performed + +var log = ctrl.Log.WithName("provider").WithName("yandex").WithName("lockbox") + +type iamTokenKey struct { + authorizedKeyID string + serviceAccountID string + privateKeyHash string +} + +// lockboxProvider is a provider for Yandex Lockbox. +type lockboxProvider struct { + yandexCloudCreator client.YandexCloudCreator + + lockboxClientMap map[string]client.LockboxClient // apiEndpoint -> LockboxClient + lockboxClientMapMutex sync.Mutex + iamTokenMap map[iamTokenKey]*client.IamToken + iamTokenMapMutex sync.Mutex +} + +func newLockboxProvider(yandexCloudCreator client.YandexCloudCreator) *lockboxProvider { + return &lockboxProvider{ + yandexCloudCreator: yandexCloudCreator, + lockboxClientMap: make(map[string]client.LockboxClient), + iamTokenMap: make(map[iamTokenKey]*client.IamToken), + } +} + +// NewClient constructs a Yandex Lockbox Provider. +func (p *lockboxProvider) NewClient(ctx context.Context, store esv1alpha1.GenericStore, kube kclient.Client, namespace string) (provider.SecretsClient, error) { + storeSpec := store.GetSpec() + if storeSpec == nil || storeSpec.Provider == nil || storeSpec.Provider.YandexLockbox == nil { + return nil, fmt.Errorf("received invalid Yandex Lockbox SecretStore resource") + } + storeSpecYandexLockbox := storeSpec.Provider.YandexLockbox + + authorizedKeySecretName := storeSpecYandexLockbox.Auth.AuthorizedKey.Name + if authorizedKeySecretName == "" { + return nil, fmt.Errorf("invalid Yandex Lockbox SecretStore resource: missing AuthorizedKey Name") + } + objectKey := types.NamespacedName{ + Name: authorizedKeySecretName, + Namespace: namespace, + } + + // only ClusterStore is allowed to set namespace (and then it's required) + if store.GetObjectKind().GroupVersionKind().Kind == esv1alpha1.ClusterSecretStoreKind { + if storeSpecYandexLockbox.Auth.AuthorizedKey.Namespace == nil { + return nil, fmt.Errorf("invalid ClusterSecretStore: missing AuthorizedKey Namespace") + } + objectKey.Namespace = *storeSpecYandexLockbox.Auth.AuthorizedKey.Namespace + } + + authorizedKeySecret := &corev1.Secret{} + err := kube.Get(ctx, objectKey, authorizedKeySecret) + if err != nil { + return nil, fmt.Errorf("could not fetch AuthorizedKey secret: %w", err) + } + + authorizedKeySecretData := authorizedKeySecret.Data[storeSpecYandexLockbox.Auth.AuthorizedKey.Key] + if (authorizedKeySecretData == nil) || (len(authorizedKeySecretData) == 0) { + return nil, fmt.Errorf("missing AuthorizedKey") + } + + var authorizedKey iamkey.Key + err = json.Unmarshal(authorizedKeySecretData, &authorizedKey) + if err != nil { + return nil, fmt.Errorf("unable to unmarshal authorized key: %w", err) + } + + lockboxClient, err := p.getOrCreateLockboxClient(ctx, storeSpecYandexLockbox.APIEndpoint, &authorizedKey) + if err != nil { + return nil, fmt.Errorf("failed to create Yandex Lockbox client: %w", err) + } + + iamToken, err := p.getOrCreateIamToken(ctx, storeSpecYandexLockbox.APIEndpoint, &authorizedKey) + if err != nil { + return nil, fmt.Errorf("failed to create IAM token: %w", err) + } + + return &lockboxSecretsClient{lockboxClient, iamToken.Token}, nil +} + +func (p *lockboxProvider) getOrCreateLockboxClient(ctx context.Context, apiEndpoint string, authorizedKey *iamkey.Key) (client.LockboxClient, error) { + p.lockboxClientMapMutex.Lock() + defer p.lockboxClientMapMutex.Unlock() + + if _, ok := p.lockboxClientMap[apiEndpoint]; !ok { + log.Info("creating LockboxClient", "apiEndpoint", apiEndpoint) + + lockboxClient, err := p.yandexCloudCreator.CreateLockboxClient(ctx, apiEndpoint, authorizedKey) + if err != nil { + return nil, err + } + p.lockboxClientMap[apiEndpoint] = lockboxClient + } + return p.lockboxClientMap[apiEndpoint], nil +} + +func (p *lockboxProvider) getOrCreateIamToken(ctx context.Context, apiEndpoint string, authorizedKey *iamkey.Key) (*client.IamToken, error) { + p.iamTokenMapMutex.Lock() + defer p.iamTokenMapMutex.Unlock() + + iamTokenKey := buildIamTokenKey(authorizedKey) + if iamToken, ok := p.iamTokenMap[iamTokenKey]; !ok || !p.isIamTokenUsable(iamToken) { + log.Info("creating IAM token", "authorizedKeyId", authorizedKey.Id) + + iamToken, err := p.yandexCloudCreator.CreateIamToken(ctx, apiEndpoint, authorizedKey) + if err != nil { + return nil, err + } + + log.Info("created IAM token", "authorizedKeyId", authorizedKey.Id, "expiresAt", iamToken.ExpiresAt) + + p.iamTokenMap[iamTokenKey] = iamToken + } + return p.iamTokenMap[iamTokenKey], nil +} + +func (p *lockboxProvider) isIamTokenUsable(iamToken *client.IamToken) bool { + now := p.yandexCloudCreator.Now() + return now.Add(maxSecretsClientLifetime).Before(iamToken.ExpiresAt) +} + +func buildIamTokenKey(authorizedKey *iamkey.Key) iamTokenKey { + privateKeyHash := sha256.Sum256([]byte(authorizedKey.PrivateKey)) + return iamTokenKey{ + authorizedKey.GetId(), + authorizedKey.GetServiceAccountId(), + hex.EncodeToString(privateKeyHash[:]), + } +} + +// Used for testing. +func (p *lockboxProvider) isIamTokenCached(authorizedKey *iamkey.Key) bool { + p.iamTokenMapMutex.Lock() + defer p.iamTokenMapMutex.Unlock() + + _, ok := p.iamTokenMap[buildIamTokenKey(authorizedKey)] + return ok +} + +func (p *lockboxProvider) cleanUpIamTokenMap() { + p.iamTokenMapMutex.Lock() + defer p.iamTokenMapMutex.Unlock() + + for key, value := range p.iamTokenMap { + if p.yandexCloudCreator.Now().After(value.ExpiresAt) { + log.Info("deleting IAM token", "authorizedKeyId", key.authorizedKeyID) + delete(p.iamTokenMap, key) + } + } +} + +// lockboxSecretsClient is a secrets client for Yandex Lockbox. +type lockboxSecretsClient struct { + lockboxClient client.LockboxClient + iamToken string +} + +// GetSecret returns a single secret from the provider. +func (c *lockboxSecretsClient) GetSecret(ctx context.Context, ref esv1alpha1.ExternalSecretDataRemoteRef) ([]byte, error) { + entries, err := c.lockboxClient.GetPayloadEntries(ctx, c.iamToken, ref.Key, ref.Version) + if err != nil { + return nil, fmt.Errorf("unable to request secret payload to get secret: %w", err) + } + + if ref.Property == "" { + keyToValue := make(map[string]interface{}, len(entries)) + for _, entry := range entries { + value, err := getValueAsIs(entry) + if err != nil { + return nil, err + } + keyToValue[entry.Key] = value + } + out, err := json.Marshal(keyToValue) + if err != nil { + return nil, fmt.Errorf("failed to marshal secret: %w", err) + } + return out, nil + } + + entry, err := findEntryByKey(entries, ref.Property) + if err != nil { + return nil, err + } + return getValueAsBinary(entry) +} + +// GetSecretMap returns multiple k/v pairs from the provider. +func (c *lockboxSecretsClient) GetSecretMap(ctx context.Context, ref esv1alpha1.ExternalSecretDataRemoteRef) (map[string][]byte, error) { + entries, err := c.lockboxClient.GetPayloadEntries(ctx, c.iamToken, ref.Key, ref.Version) + if err != nil { + return nil, fmt.Errorf("unable to request secret payload to get secret map: %w", err) + } + + secretMap := make(map[string][]byte, len(entries)) + for _, entry := range entries { + value, err := getValueAsBinary(entry) + if err != nil { + return nil, err + } + secretMap[entry.Key] = value + } + return secretMap, nil +} + +func (c *lockboxSecretsClient) Close(ctx context.Context) error { + return nil +} + +func getValueAsIs(entry *lockbox.Payload_Entry) (interface{}, error) { + switch entry.Value.(type) { + case *lockbox.Payload_Entry_TextValue: + return entry.GetTextValue(), nil + case *lockbox.Payload_Entry_BinaryValue: + return entry.GetBinaryValue(), nil + default: + return nil, fmt.Errorf("unsupported payload value type, key: %v", entry.Key) + } +} + +func getValueAsBinary(entry *lockbox.Payload_Entry) ([]byte, error) { + switch entry.Value.(type) { + case *lockbox.Payload_Entry_TextValue: + return []byte(entry.GetTextValue()), nil + case *lockbox.Payload_Entry_BinaryValue: + return entry.GetBinaryValue(), nil + default: + return nil, fmt.Errorf("unsupported payload value type, key: %v", entry.Key) + } +} + +func findEntryByKey(entries []*lockbox.Payload_Entry, key string) (*lockbox.Payload_Entry, error) { + for i := range entries { + if entries[i].Key == key { + return entries[i], nil + } + } + return nil, fmt.Errorf("payload entry with key '%s' not found", key) +} + +func init() { + lockboxProvider := newLockboxProvider(&grpc.YandexCloudCreator{}) + + go func() { + for { + time.Sleep(iamTokenCleanupDelay) + lockboxProvider.cleanUpIamTokenMap() + } + }() + + schema.Register( + lockboxProvider, + &esv1alpha1.SecretStoreProvider{ + YandexLockbox: &esv1alpha1.YandexLockboxProvider{}, + }, + ) +} diff --git a/pkg/provider/yandex/lockbox/lockbox_test.go b/pkg/provider/yandex/lockbox/lockbox_test.go new file mode 100644 index 000000000..d2ac2b602 --- /dev/null +++ b/pkg/provider/yandex/lockbox/lockbox_test.go @@ -0,0 +1,677 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package lockbox + +import ( + "context" + b64 "encoding/base64" + "encoding/json" + "testing" + "time" + + "github.com/google/uuid" + tassert "github.com/stretchr/testify/assert" + "github.com/yandex-cloud/go-genproto/yandex/cloud/lockbox/v1" + "github.com/yandex-cloud/go-sdk/iamkey" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + + esv1alpha1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1alpha1" + esmeta "github.com/external-secrets/external-secrets/apis/meta/v1" + "github.com/external-secrets/external-secrets/pkg/provider/schema" + "github.com/external-secrets/external-secrets/pkg/provider/yandex/lockbox/client/fake" +) + +func TestNewClient(t *testing.T) { + ctx := context.Background() + const namespace = "namespace" + + store := &esv1alpha1.SecretStore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + }, + Spec: esv1alpha1.SecretStoreSpec{ + Provider: &esv1alpha1.SecretStoreProvider{ + YandexLockbox: &esv1alpha1.YandexLockboxProvider{}, + }, + }, + } + provider, err := schema.GetProvider(store) + tassert.Nil(t, err) + + k8sClient := clientfake.NewClientBuilder().Build() + secretClient, err := provider.NewClient(context.Background(), store, k8sClient, namespace) + tassert.EqualError(t, err, "invalid Yandex Lockbox SecretStore resource: missing AuthorizedKey Name") + tassert.Nil(t, secretClient) + + store.Spec.Provider.YandexLockbox.Auth = esv1alpha1.YandexLockboxAuth{} + secretClient, err = provider.NewClient(context.Background(), store, k8sClient, namespace) + tassert.EqualError(t, err, "invalid Yandex Lockbox SecretStore resource: missing AuthorizedKey Name") + tassert.Nil(t, secretClient) + + store.Spec.Provider.YandexLockbox.Auth.AuthorizedKey = esmeta.SecretKeySelector{} + secretClient, err = provider.NewClient(context.Background(), store, k8sClient, namespace) + tassert.EqualError(t, err, "invalid Yandex Lockbox SecretStore resource: missing AuthorizedKey Name") + tassert.Nil(t, secretClient) + + const authorizedKeySecretName = "authorizedKeySecretName" + const authorizedKeySecretKey = "authorizedKeySecretKey" + store.Spec.Provider.YandexLockbox.Auth.AuthorizedKey.Name = authorizedKeySecretName + store.Spec.Provider.YandexLockbox.Auth.AuthorizedKey.Key = authorizedKeySecretKey + secretClient, err = provider.NewClient(context.Background(), store, k8sClient, namespace) + tassert.EqualError(t, err, "could not fetch AuthorizedKey secret: secrets \"authorizedKeySecretName\" not found") + tassert.Nil(t, secretClient) + + err = createK8sSecret(ctx, k8sClient, namespace, authorizedKeySecretName, authorizedKeySecretKey, newFakeAuthorizedKey()) + tassert.Nil(t, err) + secretClient, err = provider.NewClient(context.Background(), store, k8sClient, namespace) + tassert.EqualError(t, err, "failed to create Yandex Lockbox client: private key parsing failed: Invalid Key: Key must be PEM encoded PKCS1 or PKCS8 private key") + tassert.Nil(t, secretClient) +} + +func TestGetSecretForAllEntries(t *testing.T) { + ctx := context.Background() + namespace := uuid.NewString() + authorizedKey := newFakeAuthorizedKey() + + lockboxBackend := fake.NewLockboxBackend(time.Hour) + k1, v1 := "k1", "v1" + k2, v2 := "k2", []byte("v2") + secretID, _ := lockboxBackend.CreateSecret(authorizedKey, + textEntry(k1, v1), + binaryEntry(k2, v2), + ) + + k8sClient := clientfake.NewClientBuilder().Build() + const authorizedKeySecretName = "authorizedKeySecretName" + const authorizedKeySecretKey = "authorizedKeySecretKey" + err := createK8sSecret(ctx, k8sClient, namespace, authorizedKeySecretName, authorizedKeySecretKey, authorizedKey) + tassert.Nil(t, err) + store := newYandexLockboxSecretStore("", namespace, authorizedKeySecretName, authorizedKeySecretKey) + + provider := newLockboxProvider(&fake.YandexCloudCreator{ + Backend: lockboxBackend, + }) + secretsClient, err := provider.NewClient(ctx, store, k8sClient, namespace) + tassert.Nil(t, err) + data, err := secretsClient.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID}) + tassert.Nil(t, err) + + tassert.Equal( + t, + map[string]string{ + k1: v1, + k2: base64(v2), + }, + unmarshalStringMap(t, data), + ) +} + +func TestGetSecretForTextEntry(t *testing.T) { + ctx := context.Background() + namespace := uuid.NewString() + authorizedKey := newFakeAuthorizedKey() + + lockboxBackend := fake.NewLockboxBackend(time.Hour) + k1, v1 := "k1", "v1" + k2, v2 := "k2", []byte("v2") + secretID, _ := lockboxBackend.CreateSecret(authorizedKey, + textEntry(k1, v1), + binaryEntry(k2, v2), + ) + + k8sClient := clientfake.NewClientBuilder().Build() + const authorizedKeySecretName = "authorizedKeySecretName" + const authorizedKeySecretKey = "authorizedKeySecretKey" + err := createK8sSecret(ctx, k8sClient, namespace, authorizedKeySecretName, authorizedKeySecretKey, authorizedKey) + tassert.Nil(t, err) + store := newYandexLockboxSecretStore("", namespace, authorizedKeySecretName, authorizedKeySecretKey) + + provider := newLockboxProvider(&fake.YandexCloudCreator{ + Backend: lockboxBackend, + }) + secretsClient, err := provider.NewClient(ctx, store, k8sClient, namespace) + tassert.Nil(t, err) + data, err := secretsClient.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID, Property: k1}) + tassert.Nil(t, err) + + tassert.Equal(t, v1, string(data)) +} + +func TestGetSecretForBinaryEntry(t *testing.T) { + ctx := context.Background() + namespace := uuid.NewString() + authorizedKey := newFakeAuthorizedKey() + + lockboxBackend := fake.NewLockboxBackend(time.Hour) + k1, v1 := "k1", "v1" + k2, v2 := "k2", []byte("v2") + secretID, _ := lockboxBackend.CreateSecret(authorizedKey, + textEntry(k1, v1), + binaryEntry(k2, v2), + ) + + k8sClient := clientfake.NewClientBuilder().Build() + const authorizedKeySecretName = "authorizedKeySecretName" + const authorizedKeySecretKey = "authorizedKeySecretKey" + err := createK8sSecret(ctx, k8sClient, namespace, authorizedKeySecretName, authorizedKeySecretKey, authorizedKey) + tassert.Nil(t, err) + store := newYandexLockboxSecretStore("", namespace, authorizedKeySecretName, authorizedKeySecretKey) + + provider := newLockboxProvider(&fake.YandexCloudCreator{ + Backend: lockboxBackend, + }) + secretsClient, err := provider.NewClient(ctx, store, k8sClient, namespace) + tassert.Nil(t, err) + data, err := secretsClient.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID, Property: k2}) + tassert.Nil(t, err) + + tassert.Equal(t, v2, data) +} + +func TestGetSecretByVersionID(t *testing.T) { + ctx := context.Background() + namespace := uuid.NewString() + authorizedKey := newFakeAuthorizedKey() + + lockboxBackend := fake.NewLockboxBackend(time.Hour) + oldKey, oldVal := "oldKey", "oldVal" + secretID, oldVersionID := lockboxBackend.CreateSecret(authorizedKey, + textEntry(oldKey, oldVal), + ) + + k8sClient := clientfake.NewClientBuilder().Build() + const authorizedKeySecretName = "authorizedKeySecretName" + const authorizedKeySecretKey = "authorizedKeySecretKey" + err := createK8sSecret(ctx, k8sClient, namespace, authorizedKeySecretName, authorizedKeySecretKey, authorizedKey) + tassert.Nil(t, err) + store := newYandexLockboxSecretStore("", namespace, authorizedKeySecretName, authorizedKeySecretKey) + + provider := newLockboxProvider(&fake.YandexCloudCreator{ + Backend: lockboxBackend, + }) + secretsClient, err := provider.NewClient(ctx, store, k8sClient, namespace) + tassert.Nil(t, err) + data, err := secretsClient.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID, Version: oldVersionID}) + tassert.Nil(t, err) + + tassert.Equal(t, map[string]string{oldKey: oldVal}, unmarshalStringMap(t, data)) + + newKey, newVal := "newKey", "newVal" + newVersionID := lockboxBackend.AddVersion(secretID, + textEntry(newKey, newVal), + ) + + data, err = secretsClient.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID, Version: oldVersionID}) + tassert.Nil(t, err) + tassert.Equal(t, map[string]string{oldKey: oldVal}, unmarshalStringMap(t, data)) + + data, err = secretsClient.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID, Version: newVersionID}) + tassert.Nil(t, err) + tassert.Equal(t, map[string]string{newKey: newVal}, unmarshalStringMap(t, data)) +} + +func TestGetSecretUnauthorized(t *testing.T) { + ctx := context.Background() + namespace := uuid.NewString() + authorizedKeyA := newFakeAuthorizedKey() + authorizedKeyB := newFakeAuthorizedKey() + + lockboxBackend := fake.NewLockboxBackend(time.Hour) + secretID, _ := lockboxBackend.CreateSecret(authorizedKeyA, + textEntry("k1", "v1"), + ) + + k8sClient := clientfake.NewClientBuilder().Build() + const authorizedKeySecretName = "authorizedKeySecretName" + const authorizedKeySecretKey = "authorizedKeySecretKey" + err := createK8sSecret(ctx, k8sClient, namespace, authorizedKeySecretName, authorizedKeySecretKey, authorizedKeyB) + tassert.Nil(t, err) + store := newYandexLockboxSecretStore("", namespace, authorizedKeySecretName, authorizedKeySecretKey) + + provider := newLockboxProvider(&fake.YandexCloudCreator{ + Backend: lockboxBackend, + }) + secretsClient, err := provider.NewClient(ctx, store, k8sClient, namespace) + tassert.Nil(t, err) + _, err = secretsClient.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID}) + tassert.EqualError(t, err, "unable to request secret payload to get secret: permission denied") +} + +func TestGetSecretNotFound(t *testing.T) { + ctx := context.Background() + namespace := uuid.NewString() + authorizedKey := newFakeAuthorizedKey() + + lockboxBackend := fake.NewLockboxBackend(time.Hour) + + k8sClient := clientfake.NewClientBuilder().Build() + const authorizedKeySecretName = "authorizedKeySecretName" + const authorizedKeySecretKey = "authorizedKeySecretKey" + err := createK8sSecret(ctx, k8sClient, namespace, authorizedKeySecretName, authorizedKeySecretKey, authorizedKey) + tassert.Nil(t, err) + store := newYandexLockboxSecretStore("", namespace, authorizedKeySecretName, authorizedKeySecretKey) + + provider := newLockboxProvider(&fake.YandexCloudCreator{ + Backend: lockboxBackend, + }) + secretsClient, err := provider.NewClient(ctx, store, k8sClient, namespace) + tassert.Nil(t, err) + _, err = secretsClient.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: "no-secret-with-this-id"}) + tassert.EqualError(t, err, "unable to request secret payload to get secret: secret not found") + + secretID, _ := lockboxBackend.CreateSecret(authorizedKey, + textEntry("k1", "v1"), + ) + _, err = secretsClient.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID, Version: "no-version-with-this-id"}) + tassert.EqualError(t, err, "unable to request secret payload to get secret: version not found") +} + +func TestGetSecretWithTwoNamespaces(t *testing.T) { + ctx := context.Background() + namespace1 := uuid.NewString() + namespace2 := uuid.NewString() + authorizedKey1 := newFakeAuthorizedKey() + authorizedKey2 := newFakeAuthorizedKey() + + lockboxBackend := fake.NewLockboxBackend(time.Hour) + k1, v1 := "k1", "v1" + secretID1, _ := lockboxBackend.CreateSecret(authorizedKey1, + textEntry(k1, v1), + ) + k2, v2 := "k2", "v2" + secretID2, _ := lockboxBackend.CreateSecret(authorizedKey2, + textEntry(k2, v2), + ) + + k8sClient := clientfake.NewClientBuilder().Build() + const authorizedKeySecretName = "authorizedKeySecretName" + const authorizedKeySecretKey = "authorizedKeySecretKey" + err := createK8sSecret(ctx, k8sClient, namespace1, authorizedKeySecretName, authorizedKeySecretKey, authorizedKey1) + tassert.Nil(t, err) + err = createK8sSecret(ctx, k8sClient, namespace2, authorizedKeySecretName, authorizedKeySecretKey, authorizedKey2) + tassert.Nil(t, err) + store1 := newYandexLockboxSecretStore("", namespace1, authorizedKeySecretName, authorizedKeySecretKey) + store2 := newYandexLockboxSecretStore("", namespace2, authorizedKeySecretName, authorizedKeySecretKey) + + provider := newLockboxProvider(&fake.YandexCloudCreator{ + Backend: lockboxBackend, + }) + secretsClient1, err := provider.NewClient(ctx, store1, k8sClient, namespace1) + tassert.Nil(t, err) + secretsClient2, err := provider.NewClient(ctx, store2, k8sClient, namespace2) + tassert.Nil(t, err) + + data, err := secretsClient1.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID1, Property: k1}) + tassert.Equal(t, v1, string(data)) + tassert.Nil(t, err) + data, err = secretsClient1.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID2, Property: k2}) + tassert.Nil(t, data) + tassert.EqualError(t, err, "unable to request secret payload to get secret: permission denied") + + data, err = secretsClient2.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID1, Property: k1}) + tassert.Nil(t, data) + tassert.EqualError(t, err, "unable to request secret payload to get secret: permission denied") + data, err = secretsClient2.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID2, Property: k2}) + tassert.Equal(t, v2, string(data)) + tassert.Nil(t, err) +} + +func TestGetSecretWithTwoApiEndpoints(t *testing.T) { + ctx := context.Background() + apiEndpoint1 := uuid.NewString() + apiEndpoint2 := uuid.NewString() + namespace := uuid.NewString() + authorizedKey1 := newFakeAuthorizedKey() + authorizedKey2 := newFakeAuthorizedKey() + + lockboxBackend1 := fake.NewLockboxBackend(time.Hour) + k1, v1 := "k1", "v1" + secretID1, _ := lockboxBackend1.CreateSecret(authorizedKey1, + textEntry(k1, v1), + ) + lockboxBackend2 := fake.NewLockboxBackend(time.Hour) + k2, v2 := "k2", "v2" + secretID2, _ := lockboxBackend2.CreateSecret(authorizedKey2, + textEntry(k2, v2), + ) + + k8sClient := clientfake.NewClientBuilder().Build() + const authorizedKeySecretName1 = "authorizedKeySecretName1" + const authorizedKeySecretKey1 = "authorizedKeySecretKey1" + err := createK8sSecret(ctx, k8sClient, namespace, authorizedKeySecretName1, authorizedKeySecretKey1, authorizedKey1) + tassert.Nil(t, err) + const authorizedKeySecretName2 = "authorizedKeySecretName2" + const authorizedKeySecretKey2 = "authorizedKeySecretKey2" + err = createK8sSecret(ctx, k8sClient, namespace, authorizedKeySecretName2, authorizedKeySecretKey2, authorizedKey2) + tassert.Nil(t, err) + + store1 := newYandexLockboxSecretStore(apiEndpoint1, namespace, authorizedKeySecretName1, authorizedKeySecretKey1) + store2 := newYandexLockboxSecretStore(apiEndpoint2, namespace, authorizedKeySecretName2, authorizedKeySecretKey2) + + provider1 := newLockboxProvider(&fake.YandexCloudCreator{ + Backend: lockboxBackend1, + }) + provider2 := newLockboxProvider(&fake.YandexCloudCreator{ + Backend: lockboxBackend2, + }) + + secretsClient1, err := provider1.NewClient(ctx, store1, k8sClient, namespace) + tassert.Nil(t, err) + secretsClient2, err := provider2.NewClient(ctx, store2, k8sClient, namespace) + tassert.Nil(t, err) + + var data []byte + + data, err = secretsClient1.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID1, Property: k1}) + tassert.Equal(t, v1, string(data)) + tassert.Nil(t, err) + data, err = secretsClient1.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID2, Property: k2}) + tassert.Nil(t, data) + tassert.EqualError(t, err, "unable to request secret payload to get secret: secret not found") + + data, err = secretsClient2.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID1, Property: k1}) + tassert.Nil(t, data) + tassert.EqualError(t, err, "unable to request secret payload to get secret: secret not found") + data, err = secretsClient2.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID2, Property: k2}) + tassert.Equal(t, v2, string(data)) + tassert.Nil(t, err) +} + +func TestGetSecretWithIamTokenExpiration(t *testing.T) { + ctx := context.Background() + namespace := uuid.NewString() + authorizedKey := newFakeAuthorizedKey() + + tokenExpirationTime := time.Hour + lockboxBackend := fake.NewLockboxBackend(tokenExpirationTime) + k1, v1 := "k1", "v1" + secretID, _ := lockboxBackend.CreateSecret(authorizedKey, + textEntry(k1, v1), + ) + + k8sClient := clientfake.NewClientBuilder().Build() + const authorizedKeySecretName = "authorizedKeySecretName" + const authorizedKeySecretKey = "authorizedKeySecretKey" + err := createK8sSecret(ctx, k8sClient, namespace, authorizedKeySecretName, authorizedKeySecretKey, authorizedKey) + tassert.Nil(t, err) + store := newYandexLockboxSecretStore("", namespace, authorizedKeySecretName, authorizedKeySecretKey) + + provider := newLockboxProvider(&fake.YandexCloudCreator{ + Backend: lockboxBackend, + }) + + var data []byte + + oldSecretsClient, err := provider.NewClient(ctx, store, k8sClient, namespace) + tassert.Nil(t, err) + data, err = oldSecretsClient.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID, Property: k1}) + tassert.Equal(t, v1, string(data)) + tassert.Nil(t, err) + + lockboxBackend.AdvanceClock(2 * tokenExpirationTime) + + data, err = oldSecretsClient.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID, Property: k1}) + tassert.Nil(t, data) + tassert.EqualError(t, err, "unable to request secret payload to get secret: iam token expired") + + newSecretsClient, err := provider.NewClient(ctx, store, k8sClient, namespace) + tassert.Nil(t, err) + data, err = newSecretsClient.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID, Property: k1}) + tassert.Equal(t, v1, string(data)) + tassert.Nil(t, err) +} + +func TestGetSecretWithIamTokenCleanup(t *testing.T) { + ctx := context.Background() + namespace := uuid.NewString() + authorizedKey1 := newFakeAuthorizedKey() + authorizedKey2 := newFakeAuthorizedKey() + + tokenExpirationDuration := time.Hour + lockboxBackend := fake.NewLockboxBackend(tokenExpirationDuration) + secretID1, _ := lockboxBackend.CreateSecret(authorizedKey1, + textEntry("k1", "v1"), + ) + secretID2, _ := lockboxBackend.CreateSecret(authorizedKey2, + textEntry("k2", "v2"), + ) + + var err error + + k8sClient := clientfake.NewClientBuilder().Build() + const authorizedKeySecretName1 = "authorizedKeySecretName1" + const authorizedKeySecretKey1 = "authorizedKeySecretKey1" + err = createK8sSecret(ctx, k8sClient, namespace, authorizedKeySecretName1, authorizedKeySecretKey1, authorizedKey1) + tassert.Nil(t, err) + const authorizedKeySecretName2 = "authorizedKeySecretName2" + const authorizedKeySecretKey2 = "authorizedKeySecretKey2" + err = createK8sSecret(ctx, k8sClient, namespace, authorizedKeySecretName2, authorizedKeySecretKey2, authorizedKey2) + tassert.Nil(t, err) + + store1 := newYandexLockboxSecretStore("", namespace, authorizedKeySecretName1, authorizedKeySecretKey1) + store2 := newYandexLockboxSecretStore("", namespace, authorizedKeySecretName2, authorizedKeySecretKey2) + + provider := newLockboxProvider(&fake.YandexCloudCreator{ + Backend: lockboxBackend, + }) + + tassert.False(t, provider.isIamTokenCached(authorizedKey1)) + tassert.False(t, provider.isIamTokenCached(authorizedKey2)) + + // Access secretID1 with authorizedKey1, IAM token for authorizedKey1 should be cached + secretsClient, err := provider.NewClient(ctx, store1, k8sClient, namespace) + tassert.Nil(t, err) + _, err = secretsClient.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID1}) + tassert.Nil(t, err) + + tassert.True(t, provider.isIamTokenCached(authorizedKey1)) + tassert.False(t, provider.isIamTokenCached(authorizedKey2)) + + lockboxBackend.AdvanceClock(tokenExpirationDuration * 2) + + // Access secretID2 with authorizedKey2, IAM token for authorizedKey2 should be cached + secretsClient, err = provider.NewClient(ctx, store2, k8sClient, namespace) + tassert.Nil(t, err) + _, err = secretsClient.GetSecret(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID2}) + tassert.Nil(t, err) + + tassert.True(t, provider.isIamTokenCached(authorizedKey1)) + tassert.True(t, provider.isIamTokenCached(authorizedKey2)) + + lockboxBackend.AdvanceClock(tokenExpirationDuration) + + tassert.True(t, provider.isIamTokenCached(authorizedKey1)) + tassert.True(t, provider.isIamTokenCached(authorizedKey2)) + + provider.cleanUpIamTokenMap() + + tassert.False(t, provider.isIamTokenCached(authorizedKey1)) + tassert.True(t, provider.isIamTokenCached(authorizedKey2)) + + lockboxBackend.AdvanceClock(tokenExpirationDuration) + + tassert.False(t, provider.isIamTokenCached(authorizedKey1)) + tassert.True(t, provider.isIamTokenCached(authorizedKey2)) + + provider.cleanUpIamTokenMap() + + tassert.False(t, provider.isIamTokenCached(authorizedKey1)) + tassert.False(t, provider.isIamTokenCached(authorizedKey2)) +} + +func TestGetSecretMap(t *testing.T) { + ctx := context.Background() + namespace := uuid.NewString() + authorizedKey := newFakeAuthorizedKey() + + lockboxBackend := fake.NewLockboxBackend(time.Hour) + k1, v1 := "k1", "v1" + k2, v2 := "k2", []byte("v2") + secretID, _ := lockboxBackend.CreateSecret(authorizedKey, + textEntry(k1, v1), + binaryEntry(k2, v2), + ) + + k8sClient := clientfake.NewClientBuilder().Build() + const authorizedKeySecretName = "authorizedKeySecretName" + const authorizedKeySecretKey = "authorizedKeySecretKey" + err := createK8sSecret(ctx, k8sClient, namespace, authorizedKeySecretName, authorizedKeySecretKey, authorizedKey) + tassert.Nil(t, err) + store := newYandexLockboxSecretStore("", namespace, authorizedKeySecretName, authorizedKeySecretKey) + + provider := newLockboxProvider(&fake.YandexCloudCreator{ + Backend: lockboxBackend, + }) + secretsClient, err := provider.NewClient(ctx, store, k8sClient, namespace) + tassert.Nil(t, err) + data, err := secretsClient.GetSecretMap(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID}) + tassert.Nil(t, err) + + tassert.Equal( + t, + map[string][]byte{ + k1: []byte(v1), + k2: v2, + }, + data, + ) +} + +func TestGetSecretMapByVersionID(t *testing.T) { + ctx := context.Background() + namespace := uuid.NewString() + authorizedKey := newFakeAuthorizedKey() + + lockboxBackend := fake.NewLockboxBackend(time.Hour) + oldKey, oldVal := "oldKey", "oldVal" + secretID, oldVersionID := lockboxBackend.CreateSecret(authorizedKey, + textEntry(oldKey, oldVal), + ) + + k8sClient := clientfake.NewClientBuilder().Build() + const authorizedKeySecretName = "authorizedKeySecretName" + const authorizedKeySecretKey = "authorizedKeySecretKey" + err := createK8sSecret(ctx, k8sClient, namespace, authorizedKeySecretName, authorizedKeySecretKey, authorizedKey) + tassert.Nil(t, err) + store := newYandexLockboxSecretStore("", namespace, authorizedKeySecretName, authorizedKeySecretKey) + + provider := newLockboxProvider(&fake.YandexCloudCreator{ + Backend: lockboxBackend, + }) + secretsClient, err := provider.NewClient(ctx, store, k8sClient, namespace) + tassert.Nil(t, err) + data, err := secretsClient.GetSecretMap(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID, Version: oldVersionID}) + tassert.Nil(t, err) + + tassert.Equal(t, map[string][]byte{oldKey: []byte(oldVal)}, data) + + newKey, newVal := "newKey", "newVal" + newVersionID := lockboxBackend.AddVersion(secretID, + textEntry(newKey, newVal), + ) + + data, err = secretsClient.GetSecretMap(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID, Version: oldVersionID}) + tassert.Nil(t, err) + tassert.Equal(t, map[string][]byte{oldKey: []byte(oldVal)}, data) + + data, err = secretsClient.GetSecretMap(ctx, esv1alpha1.ExternalSecretDataRemoteRef{Key: secretID, Version: newVersionID}) + tassert.Nil(t, err) + tassert.Equal(t, map[string][]byte{newKey: []byte(newVal)}, data) +} + +// helper functions + +func newYandexLockboxSecretStore(apiEndpoint, namespace, authorizedKeySecretName, authorizedKeySecretKey string) esv1alpha1.GenericStore { + return &esv1alpha1.SecretStore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + }, + Spec: esv1alpha1.SecretStoreSpec{ + Provider: &esv1alpha1.SecretStoreProvider{ + YandexLockbox: &esv1alpha1.YandexLockboxProvider{ + APIEndpoint: apiEndpoint, + Auth: esv1alpha1.YandexLockboxAuth{ + AuthorizedKey: esmeta.SecretKeySelector{ + Name: authorizedKeySecretName, + Key: authorizedKeySecretKey, + }, + }, + }, + }, + }, + } +} + +func createK8sSecret(ctx context.Context, k8sClient client.Client, namespace, secretName, secretKey string, secretContent interface{}) error { + data, err := json.Marshal(secretContent) + if err != nil { + return err + } + + err = k8sClient.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: secretName, + }, + Data: map[string][]byte{secretKey: data}, + }) + if err != nil { + return err + } + + return nil +} + +func newFakeAuthorizedKey() *iamkey.Key { + uniqueLabel := uuid.NewString() + return &iamkey.Key{ + Id: uniqueLabel, + Subject: &iamkey.Key_ServiceAccountId{ + ServiceAccountId: uniqueLabel, + }, + PrivateKey: uniqueLabel, + } +} + +func textEntry(key, value string) *lockbox.Payload_Entry { + return &lockbox.Payload_Entry{ + Key: key, + Value: &lockbox.Payload_Entry_TextValue{ + TextValue: value, + }, + } +} + +func binaryEntry(key string, value []byte) *lockbox.Payload_Entry { + return &lockbox.Payload_Entry{ + Key: key, + Value: &lockbox.Payload_Entry_BinaryValue{ + BinaryValue: value, + }, + } +} + +func unmarshalStringMap(t *testing.T, data []byte) map[string]string { + stringMap := make(map[string]string) + err := json.Unmarshal(data, &stringMap) + tassert.Nil(t, err) + return stringMap +} + +func base64(data []byte) string { + return b64.StdEncoding.EncodeToString(data) +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 23297d764..d0939dc45 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -14,14 +14,20 @@ limitations under the License. package utils -import "reflect" +import ( + + // nolint:gosec + "crypto/md5" + "fmt" + "reflect" +) // MergeByteMap merges map of byte slices. -func MergeByteMap(src, dst map[string][]byte) map[string][]byte { - for k, v := range dst { - src[k] = v +func MergeByteMap(dst, src map[string][]byte) map[string][]byte { + for k, v := range src { + dst[k] = v } - return src + return dst } // MergeStringMap performs a deep clone from src to dest. @@ -35,3 +41,10 @@ func MergeStringMap(dest, src map[string]string) { func IsNil(i interface{}) bool { return i == nil || reflect.ValueOf(i).IsNil() } + +// ObjectHash calculates md5 sum of the data contained in the secret. +// nolint:gosec +func ObjectHash(object interface{}) string { + textualVersion := fmt.Sprintf("%+v", object) + return fmt.Sprintf("%x", md5.Sum([]byte(textualVersion))) +} diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go new file mode 100644 index 000000000..c830c1a8d --- /dev/null +++ b/pkg/utils/utils_test.go @@ -0,0 +1,62 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "testing" + + v1 "k8s.io/api/core/v1" +) + +func TestObjectHash(t *testing.T) { + tests := []struct { + name string + input interface{} + want string + }{ + { + name: "A nil should be still working", + input: nil, + want: "60046f14c917c18a9a0f923e191ba0dc", + }, + { + name: "We accept a simple scalar value, i.e. string", + input: "hello there", + want: "161bc25962da8fed6d2f59922fb642aa", + }, + { + name: "A complex object like a secret is not an issue", + input: v1.Secret{Data: map[string][]byte{ + "xx": []byte("yyy"), + }}, + want: "a9fe13fd43b20829b45f0a93372413dd", + }, + { + name: "map also works", + input: map[string][]byte{ + "foo": []byte("value1"), + "bar": []byte("value2"), + }, + want: "caa0155759a6a9b3b6ada5a6883ee2bb", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ObjectHash(tt.input); got != tt.want { + t.Errorf("ObjectHash() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/tools.go b/tools.go index c73ab0b74..16d3fad6c 100644 --- a/tools.go +++ b/tools.go @@ -1,3 +1,4 @@ +//go:build tools // +build tools package tools