From 51532ca8a10f23564c8a68075a8ca191525497bb Mon Sep 17 00:00:00 2001 From: Moritz Johner Date: Sun, 29 Oct 2023 21:51:39 +0100 Subject: [PATCH] feat: add AKS e2e managed (#2811) Migrate azure e2e tests to use the new TFC_* secrets which are provisioned through external-secrets/infrastructure. Also enable the use of `/ok-to-test-managed provider=azure` command to run e2e managed tests that verify integration with AKS and Azure Workload Identity (AZWI). Signed-off-by: Moritz Johner --- .github/actions/e2e-managed/action.yml | 41 +++++++++-- .github/workflows/e2e-managed.yml | 10 +-- .github/workflows/e2e.yml | 10 ++- e2e/go.mod | 5 +- e2e/go.sum | 6 ++ e2e/run.sh | 10 +-- .../provider/cases/azure/azure_managed.go | 6 +- e2e/suites/provider/cases/azure/provider.go | 73 ++++++++++++++++--- terraform/azure/aks/providers.tf | 13 ---- terraform/azure/key-vault/main.tf | 45 ++++++++++++ terraform/azure/key-vault/providers.tf | 11 --- terraform/azure/key-vault/variables.tf | 5 ++ terraform/azure/main.tf | 61 ++++++++++++++-- terraform/azure/providers.tf | 22 +++++- terraform/azure/resource-group/main.tf | 4 - terraform/azure/resource-group/providers.tf | 11 --- terraform/azure/resource-group/variables.tf | 9 --- terraform/azure/service-principal/main.tf | 5 -- .../azure/service-principal/providers.tf | 11 --- terraform/azure/workload-identity/main.tf | 2 +- terraform/azure/workload-identity/provider.tf | 5 -- 21 files changed, 247 insertions(+), 118 deletions(-) delete mode 100644 terraform/azure/aks/providers.tf delete mode 100644 terraform/azure/key-vault/providers.tf delete mode 100644 terraform/azure/resource-group/main.tf delete mode 100644 terraform/azure/resource-group/providers.tf delete mode 100644 terraform/azure/resource-group/variables.tf delete mode 100644 terraform/azure/service-principal/providers.tf delete mode 100644 terraform/azure/workload-identity/provider.tf diff --git a/.github/actions/e2e-managed/action.yml b/.github/actions/e2e-managed/action.yml index 1e9436dfc..2f89030b8 100644 --- a/.github/actions/e2e-managed/action.yml +++ b/.github/actions/e2e-managed/action.yml @@ -37,12 +37,6 @@ runs: }); return result; - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - name: Setup Go uses: actions/setup-go@v3 with: @@ -78,6 +72,13 @@ runs: shell: bash run: find ${{ github.workspace }} | grep tf$ | xargs -n1 dirname | xargs -IXXX -n1 /bin/sh -c 'set -o errexit; cd XXX; pwd; tflint --loglevel=info .; cd - >/dev/null' + - name: Configure AWS Credentials + if: env.CLOUD_PROVIDER == 'aws' + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + - name: Setup TF Gcloud Provider shell: bash if: env.CLOUD_PROVIDER == 'gcp' @@ -87,8 +88,20 @@ runs: mkdir -p terraform/gcp/secrets echo ${GCP_SM_SA_GKE_JSON} > terraform/gcp/secrets/gcloud-service-account-key.json + - name: 'Az CLI login' + uses: azure/login@v1 + if: env.CLOUD_PROVIDER == 'azure' + with: + client-id: ${{ env.TFC_AZURE_CLIENT_ID }} + tenant-id: ${{ env.TFC_AZURE_TENANT_ID }} + subscription-id: ${{ env.TFC_AZURE_SUBSCRIPTION_ID }} + - name: Show TF shell: bash + env: + ARM_CLIENT_ID: "${{ env.TFC_AZURE_CLIENT_ID }}" + ARM_SUBSCRIPTION_ID: "${{ env.TFC_AZURE_SUBSCRIPTION_ID }}" + ARM_TENANT_ID: "${{ env.TFC_AZURE_TENANT_ID }}" run: |- PROVIDER=${{env.CLOUD_PROVIDER}} make tf.show.${PROVIDER} @@ -96,7 +109,9 @@ runs: - name: Apply TF shell: bash env: - TF_VAR_OIDC_TOKEN: "${{steps.fetch-token.outputs.result}}" + ARM_CLIENT_ID: "${{ env.TFC_AZURE_CLIENT_ID }}" + ARM_SUBSCRIPTION_ID: "${{ env.TFC_AZURE_SUBSCRIPTION_ID }}" + ARM_TENANT_ID: "${{ env.TFC_AZURE_TENANT_ID }}" run: |- PROVIDER=${{env.CLOUD_PROVIDER}} make tf.apply.${PROVIDER} @@ -120,6 +135,12 @@ runs: if: env.CLOUD_PROVIDER == 'aws' run: |- aws --region $AWS_REGION eks update-kubeconfig --name $AWS_CLUSTER_NAME + + - name: Get AKS credentials + if: env.CLOUD_PROVIDER == 'azure' + shell: bash + run: |- + az aks get-credentials --admin --name eso-cluster --resource-group external-secrets-operator - name: Login to Docker uses: docker/login-action@v2 @@ -137,11 +158,15 @@ runs: export PATH=$PATH:$(go env GOPATH)/bin PROVIDER=${{env.CLOUD_PROVIDER}} go install github.com/onsi/ginkgo/v2/ginkgo@v2.1.6 - make test.e2e.managed GINKGO_LABELS="${PROVIDER}" TEST_SUITES="provider" + make test.e2e.managed GINKGO_LABELS="${PROVIDER} && managed" TEST_SUITES="provider" - name: Destroy TF shell: bash if: always() + env: + ARM_CLIENT_ID: "${{ env.TFC_AZURE_CLIENT_ID }}" + ARM_SUBSCRIPTION_ID: "${{ env.TFC_AZURE_SUBSCRIPTION_ID }}" + ARM_TENANT_ID: "${{ env.TFC_AZURE_TENANT_ID }}" run: |- PROVIDER=${{env.CLOUD_PROVIDER}} make tf.destroy.${PROVIDER} diff --git a/.github/workflows/e2e-managed.yml b/.github/workflows/e2e-managed.yml index 1510fd508..7ade43d2c 100644 --- a/.github/workflows/e2e-managed.yml +++ b/.github/workflows/e2e-managed.yml @@ -39,10 +39,11 @@ env: TF_VAR_AWS_REGION: "eu-central-1" TF_VAR_AWS_CLUSTER_NAME: "eso-e2e-managed" - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID}} - AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET}} - TENANT_ID: ${{ secrets.TENANT_ID}} - VAULT_URL: ${{ secrets.VAULT_URL}} + TFC_AZURE_CLIENT_ID: ${{ secrets.TFC_AZURE_CLIENT_ID}} + TFC_AZURE_CLIENT_SECRET: ${{ secrets.TFC_AZURE_CLIENT_SECRET }} + TFC_AZURE_TENANT_ID: ${{ secrets.TFC_AZURE_TENANT_ID}} + TFC_AZURE_SUBSCRIPTION_ID: ${{ secrets.TFC_AZURE_SUBSCRIPTION_ID }} + TFC_VAULT_URL: ${{ secrets.TFC_VAULT_URL}} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_PR_NUMBER: ${{ github.event.client_payload.pull_request.number }} @@ -66,7 +67,6 @@ jobs: - uses: ./.github/actions/e2e-managed env: - CLOUD_PROVIDER: aws GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} integration-managed: diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 892fc5030..098ce4604 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -32,10 +32,12 @@ env: AWS_REGION: "eu-central-1" AWS_OIDC_ROLE_ARN: ${{ secrets.AWS_OIDC_ROLE_ARN }} - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID}} - AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET}} - TENANT_ID: ${{ secrets.TENANT_ID}} - VAULT_URL: ${{ secrets.VAULT_URL}} + TFC_AZURE_CLIENT_ID: ${{ secrets.TFC_AZURE_CLIENT_ID}} + TFC_AZURE_CLIENT_SECRET: ${{ secrets.TFC_AZURE_CLIENT_SECRET }} + TFC_AZURE_TENANT_ID: ${{ secrets.TFC_AZURE_TENANT_ID}} + TFC_AZURE_SUBSCRIPTION_ID: ${{ secrets.TFC_AZURE_SUBSCRIPTION_ID }} + TFC_VAULT_URL: ${{ secrets.TFC_VAULT_URL}} + SCALEWAY_API_URL: ${{ secrets.SCALEWAY_API_URL }} SCALEWAY_REGION: ${{ secrets.SCALEWAY_REGION }} SCALEWAY_PROJECT_ID: ${{ secrets.SCALEWAY_PROJECT_ID }} diff --git a/e2e/go.mod b/e2e/go.mod index 3532b7f80..c96219001 100644 --- a/e2e/go.mod +++ b/e2e/go.mod @@ -40,6 +40,7 @@ replace ( require ( cloud.google.com/go/secretmanager v1.11.2 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible + github.com/Azure/go-autorest/autorest v0.11.29 github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 github.com/DelineaXPM/dsv-sdk-go/v2 v2.1.0 github.com/akeylesslabs/akeyless-go-cloud-id v0.3.4 @@ -74,7 +75,6 @@ require ( cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.3 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest v0.11.29 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect @@ -82,6 +82,7 @@ require ( github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Masterminds/sprig/v3 v3.2.3 // indirect @@ -105,6 +106,7 @@ require ( github.com/goccy/go-json v0.10.2 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic-models v0.6.8 // indirect @@ -130,6 +132,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect diff --git a/e2e/go.sum b/e2e/go.sum index a2459db6c..eb6f4da02 100644 --- a/e2e/go.sum +++ b/e2e/go.sum @@ -76,6 +76,8 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 h1:hVeq+yCyUi+MsoO/CU95yqCIcdzra5ovzk8Q2BBpV2M= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DelineaXPM/dsv-sdk-go/v2 v2.1.0 h1:+XXJ43iH4js8LIBr4MUGq1J09ycivNkTNhtn4mFyhY8= @@ -186,6 +188,8 @@ github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= +github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -332,6 +336,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lestrrat-go/backoff/v2 v2.0.8 h1:oNb5E5isby2kiro9AgdHLv5N5tint1AnDVVf2E2un5A= github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y= github.com/lestrrat-go/blackmagic v1.0.1/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= diff --git a/e2e/run.sh b/e2e/run.sh index 9977afe46..f71c134a1 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -44,7 +44,7 @@ kubectl run --rm \ --attach \ --restart=Never \ --pod-running-timeout=5m \ - --labels="app=eso-e2e" \ + --labels="app=eso-e2e,azure.workload.identity/use=true" \ --env="ACK_GINKGO_DEPRECATIONS=2.9.5" \ --env="GINKGO_LABELS=${GINKGO_LABELS:-.*}" \ --env="GCP_SM_SA_JSON=${GCP_SM_SA_JSON:-}" \ @@ -59,13 +59,13 @@ kubectl run --rm \ --env="AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-}" \ --env="AWS_SA_NAME=${AWS_SA_NAME:-}" \ --env="AWS_SA_NAMESPACE=${AWS_SA_NAMESPACE:-}" \ - --env="AZURE_CLIENT_ID=${AZURE_CLIENT_ID:-}" \ - --env="AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET:-}" \ + --env="TFC_AZURE_CLIENT_ID=${TFC_AZURE_CLIENT_ID:-}" \ + --env="TFC_AZURE_CLIENT_SECRET=${TFC_AZURE_CLIENT_SECRET:-}" \ + --env="TFC_AZURE_TENANT_ID=${TFC_AZURE_TENANT_ID:-}" \ + --env="TFC_VAULT_URL=${TFC_VAULT_URL:-}" \ --env="AKEYLESS_ACCESS_ID=${AKEYLESS_ACCESS_ID:-}" \ --env="AKEYLESS_ACCESS_TYPE=${AKEYLESS_ACCESS_TYPE:-}" \ --env="AKEYLESS_ACCESS_TYPE_PARAM=${AKEYLESS_ACCESS_TYPE_PARAM:-}" \ - --env="TENANT_ID=${TENANT_ID:-}" \ - --env="VAULT_URL=${VAULT_URL:-}" \ --env="GITLAB_TOKEN=${GITLAB_TOKEN:-}" \ --env="GITLAB_PROJECT_ID=${GITLAB_PROJECT_ID:-}" \ --env="GITLAB_ENVIRONMENT=${GITLAB_ENVIRONMENT:-}" \ diff --git a/e2e/suites/provider/cases/azure/azure_managed.go b/e2e/suites/provider/cases/azure/azure_managed.go index 298807b9a..4608a2f4d 100644 --- a/e2e/suites/provider/cases/azure/azure_managed.go +++ b/e2e/suites/provider/cases/azure/azure_managed.go @@ -34,15 +34,15 @@ const ( // to test workload-identity authentication. var _ = Describe("[azuremanaged] with pod identity", Label("azure", "keyvault", "managed", "workload-identity"), func() { f := framework.New("eso-azuremanaged") - prov := newFromEnv(f) + prov := newFromWorkloadIdentity(f) // each test case gets its own ESO instance BeforeEach(func() { f.Install(addon.NewESO( addon.WithControllerClass(f.BaseName), - addon.WithServiceAccount(prov.clientID), addon.WithReleaseName(f.Namespace.Name), - addon.WithNamespace("default"), + addon.WithNamespace("external-secrets-operator"), + addon.WithServiceAccount("external-secrets-operator"), addon.WithoutWebhook(), addon.WithoutCertController(), )) diff --git a/e2e/suites/provider/cases/azure/provider.go b/e2e/suites/provider/cases/azure/provider.go index 5e4a03135..1c45fc75c 100644 --- a/e2e/suites/provider/cases/azure/provider.go +++ b/e2e/suites/provider/cases/azure/provider.go @@ -15,10 +15,13 @@ package azure import ( "context" "os" + "strings" "sync" "time" "github.com/Azure/azure-sdk-for-go/profiles/latest/keyvault/keyvault" + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/azure" kvauth "github.com/Azure/go-autorest/autorest/azure/auth" // nolint @@ -32,6 +35,7 @@ import ( "github.com/external-secrets/external-secrets-e2e/framework" esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1" esmeta "github.com/external-secrets/external-secrets/apis/meta/v1" + esoazkv "github.com/external-secrets/external-secrets/pkg/provider/azure/keyvault" ) type azureProvider struct { @@ -43,30 +47,38 @@ type azureProvider struct { framework *framework.Framework } -func newazureProvider(f *framework.Framework, clientID, clientSecret, tenantID, vaultURL string) *azureProvider { - clientCredentialsConfig := kvauth.NewClientCredentialsConfig(clientID, clientSecret, tenantID) - clientCredentialsConfig.Resource = "https://vault.azure.net" +// newFromEnv creates a new Azure KeyVault e2e test provider +// which uses client credentials flow to authenticate with azure. +func newFromEnv(f *framework.Framework) *azureProvider { + vaultURL := os.Getenv("TFC_VAULT_URL") + tenantID := os.Getenv("TFC_AZURE_TENANT_ID") + clientID := os.Getenv("TFC_AZURE_CLIENT_ID") + clientSecret := os.Getenv("TFC_AZURE_CLIENT_SECRET") + basicClient := keyvault.New() prov := &azureProvider{ framework: f, - client: &basicClient, clientID: clientID, - clientSecret: clientSecret, tenantID: tenantID, vaultURL: vaultURL, + client: &basicClient, + clientSecret: clientSecret, } o := &sync.Once{} BeforeEach(func() { // run authorizor only if this spec is called + // this allows us to run OTHER providers using GINKGO_LABELS without bailing out o.Do(func() { + defer GinkgoRecover() + clientCredentialsConfig := kvauth.NewClientCredentialsConfig(clientID, clientSecret, tenantID) + clientCredentialsConfig.Resource = "https://vault.azure.net" authorizer, err := clientCredentialsConfig.Authorizer() if err != nil { Fail(err.Error()) } prov.client.Authorizer = authorizer }) - prov.CreateSecretStoreWithWI() prov.CreateSecretStore() prov.CreateReferentSecretStore() }) @@ -74,12 +86,51 @@ func newazureProvider(f *framework.Framework, clientID, clientSecret, tenantID, return prov } -func newFromEnv(f *framework.Framework) *azureProvider { - vaultURL := os.Getenv("VAULT_URL") - tenantID := os.Getenv("TENANT_ID") +// create a new provider from workload identity +// the azwi webhook injects `AZURE_*` env vars into the container. +// we use these credentials to authenticate with azure using the federated token flow. +// please see here for details: https://azure.github.io/azure-workload-identity/docs/quick-start.html +func newFromWorkloadIdentity(f *framework.Framework) *azureProvider { + // from azwi webhook + tenantID := os.Getenv("AZURE_TENANT_ID") clientID := os.Getenv("AZURE_CLIENT_ID") - clientSecret := os.Getenv("AZURE_CLIENT_SECRET") - return newazureProvider(f, clientID, clientSecret, tenantID, vaultURL) + tokenFilePath := os.Getenv("AZURE_FEDERATED_TOKEN_FILE") + + // from run.sh + vaultURL := "https://eso-testing.vault.azure.net/" + + basicClient := keyvault.New() + prov := &azureProvider{ + framework: f, + client: &basicClient, + clientID: clientID, + tenantID: tenantID, + vaultURL: vaultURL, + } + + o := &sync.Once{} + BeforeEach(func() { + prov.CreateSecretStoreWithWI() + // run authorizor only if this spec is called + o.Do(func() { + defer GinkgoRecover() + token, err := os.ReadFile(tokenFilePath) + if err != nil { + Fail(err.Error()) + } + + // exchange the federated token for an access token + aadEndpoint := esoazkv.AadEndpointForType(esv1beta1.AzureEnvironmentPublicCloud) + kvResource := strings.TrimSuffix(azure.PublicCloud.KeyVaultEndpoint, "/") + tokenProvider, err := esoazkv.NewTokenProvider(context.Background(), string(token), clientID, tenantID, aadEndpoint, kvResource) + if err != nil { + Fail(err.Error()) + } + basicClient.Authorizer = autorest.NewBearerAuthorizer(tokenProvider) + }) + }) + + return prov } func (s *azureProvider) CreateSecret(key string, val framework.SecretEntry) { diff --git a/terraform/azure/aks/providers.tf b/terraform/azure/aks/providers.tf deleted file mode 100644 index 63988c35c..000000000 --- a/terraform/azure/aks/providers.tf +++ /dev/null @@ -1,13 +0,0 @@ -terraform { - required_providers { - azurerm = { - source = "hashicorp/azurerm" - } - } -} - -provider "azurerm" { - features {} -} - - diff --git a/terraform/azure/key-vault/main.tf b/terraform/azure/key-vault/main.tf index 55158dbd6..c0cbdb66e 100644 --- a/terraform/azure/key-vault/main.tf +++ b/terraform/azure/key-vault/main.tf @@ -16,6 +16,12 @@ resource "azurerm_key_vault" "current" { key_permissions = [ "Get", + "List", + "Create", + "Delete", + "Purge", + "Decrypt", + "Encrypt", ] secret_permissions = [ @@ -27,7 +33,11 @@ resource "azurerm_key_vault" "current" { ] storage_permissions = [ + "Set", "Get", + "Delete", + "Purge", + "Recover" ] } access_policy { @@ -36,7 +46,42 @@ resource "azurerm_key_vault" "current" { secret_permissions = [ "Get", + "Set", + "Delete", + "Purge", + "Recover", ] } + + access_policy { + tenant_id = var.tenant_id + object_id = var.eso_e2e_sp_object_id + + secret_permissions = [ + "Get", + "Set", + "Delete", + "Purge", + "Recover", + ] + + key_permissions = [ + "Get", + "List", + "Create", + "Delete", + "Purge", + "Decrypt", + "Encrypt", + ] + + certificate_permissions = [ + "Get", + "List", + "Create", + "Delete", + "Purge", + ] + } } diff --git a/terraform/azure/key-vault/providers.tf b/terraform/azure/key-vault/providers.tf deleted file mode 100644 index 0ca0933f0..000000000 --- a/terraform/azure/key-vault/providers.tf +++ /dev/null @@ -1,11 +0,0 @@ -terraform { - required_providers { - azurerm = { - source = "hashicorp/azurerm" - } - } -} - -provider "azurerm" { - features {} -} diff --git a/terraform/azure/key-vault/variables.tf b/terraform/azure/key-vault/variables.tf index 3903e78b6..68ab75490 100644 --- a/terraform/azure/key-vault/variables.tf +++ b/terraform/azure/key-vault/variables.tf @@ -22,3 +22,8 @@ variable "eso_sp_object_id" { type = string description = "The object ID of the ESO service account" } + +variable "eso_e2e_sp_object_id" { + type = string + description = "The object ID of the ESO e2e service account" +} diff --git a/terraform/azure/main.tf b/terraform/azure/main.tf index 5db744db8..4bd4bbbd1 100644 --- a/terraform/azure/main.tf +++ b/terraform/azure/main.tf @@ -2,11 +2,9 @@ data "azurerm_client_config" "current" {} data "azurerm_subscription" "primary" {} -module "test_resource_group" { - source = "./resource-group" - - resource_group_name = var.resource_group_name - resource_group_location = var.resource_group_location +resource "azurerm_resource_group" "current" { + name = var.resource_group_name + location = var.resource_group_location } module "test_sp" { @@ -16,6 +14,19 @@ module "test_sp" { application_owners = [data.azurerm_client_config.current.object_id] issuer = module.test_aks.cluster_issuer_url subject = "system:serviceaccount:${var.sa_namespace}:${var.sa_name}" + + depends_on = [ + azurerm_resource_group.current + ] +} + +module "e2e_sp" { + source = "./service-principal" + + application_display_name = var.application_display_name + application_owners = [data.azurerm_client_config.current.object_id] + issuer = module.test_aks.cluster_issuer_url + subject = "system:serviceaccount:default:external-secrets-e2e" } module "test_key_vault" { @@ -27,6 +38,11 @@ module "test_key_vault" { tenant_id = data.azurerm_client_config.current.tenant_id client_object_id = data.azurerm_client_config.current.object_id eso_sp_object_id = module.test_sp.sp_object_id + eso_e2e_sp_object_id = module.e2e_sp.sp_object_id + + depends_on = [ + azurerm_resource_group.current + ] } module "test_workload_identity" { @@ -46,12 +62,43 @@ module "test_aks" { default_node_pool_node_count = var.default_node_pool_node_count default_node_pool_vm_size = var.default_node_pool_vm_size cluster_tags = var.cluster_tags + + depends_on = [ + azurerm_resource_group.current + ] } resource "azurerm_role_assignment" "current" { scope = data.azurerm_subscription.primary.id - role_definition_name = "Reader" + role_definition_name = "Owner" principal_id = module.test_sp.sp_id + + depends_on = [ + azurerm_resource_group.current + ] +} + +resource "kubernetes_namespace" "eso" { + metadata { + name = "external-secrets-operator" + } +} + +// the `e2e` pod itself runs with workload identity and +// does not rely on client credentials. +resource "kubernetes_service_account" "e2e" { + metadata { + name = "external-secrets-e2e" + namespace = "default" + annotations = { + "azure.workload.identity/client-id" = module.e2e_sp.application_id + "azure.workload.identity/tenant-id" = data.azurerm_client_config.current.tenant_id + } + labels = { + "azure.workload.identity/use" = "true" + } + } + depends_on = [module.test_aks, kubernetes_namespace.eso] } resource "kubernetes_service_account" "current" { @@ -66,5 +113,5 @@ resource "kubernetes_service_account" "current" { "azure.workload.identity/use" = "true" } } - + depends_on = [module.test_aks, kubernetes_namespace.eso] } diff --git a/terraform/azure/providers.tf b/terraform/azure/providers.tf index d5f626ab7..4452576ae 100644 --- a/terraform/azure/providers.tf +++ b/terraform/azure/providers.tf @@ -8,14 +8,28 @@ terraform { provider "azurerm" { features {} + # set this to false when running locally + use_oidc = true +} + +data "azurerm_kubernetes_cluster" "default" { + depends_on = [module.test_aks] # refresh cluster state before reading + name = var.cluster_name + resource_group_name = var.resource_group_name } provider "helm" { kubernetes { - config_path = "~/.kube/config" + host = data.azurerm_kubernetes_cluster.default.kube_config.0.host + client_certificate = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.client_certificate) + client_key = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.client_key) + cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.cluster_ca_certificate) } } -provider "kubernetes" { - config_path = "~/.kube/config" -} +provider "kubernetes" { + host = data.azurerm_kubernetes_cluster.default.kube_config.0.host + client_certificate = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.client_certificate) + client_key = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.client_key) + cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.cluster_ca_certificate) +} diff --git a/terraform/azure/resource-group/main.tf b/terraform/azure/resource-group/main.tf deleted file mode 100644 index cbdd17429..000000000 --- a/terraform/azure/resource-group/main.tf +++ /dev/null @@ -1,4 +0,0 @@ -resource "azurerm_resource_group" "current" { - name = var.resource_group_name - location = var.resource_group_location -} diff --git a/terraform/azure/resource-group/providers.tf b/terraform/azure/resource-group/providers.tf deleted file mode 100644 index 0ca0933f0..000000000 --- a/terraform/azure/resource-group/providers.tf +++ /dev/null @@ -1,11 +0,0 @@ -terraform { - required_providers { - azurerm = { - source = "hashicorp/azurerm" - } - } -} - -provider "azurerm" { - features {} -} diff --git a/terraform/azure/resource-group/variables.tf b/terraform/azure/resource-group/variables.tf deleted file mode 100644 index 752fb124c..000000000 --- a/terraform/azure/resource-group/variables.tf +++ /dev/null @@ -1,9 +0,0 @@ -variable "resource_group_name" { - type = string - description = "The Name which should be used for this Resource Group" -} - -variable "resource_group_location" { - type = string - description = "The Azure Region where the Resource Group should exist" -} diff --git a/terraform/azure/service-principal/main.tf b/terraform/azure/service-principal/main.tf index 9dd28060a..f9a27ef6e 100644 --- a/terraform/azure/service-principal/main.tf +++ b/terraform/azure/service-principal/main.tf @@ -1,11 +1,9 @@ resource "azuread_application" "current" { - display_name = var.application_display_name owners = var.application_owners } resource "azuread_service_principal" "current" { - application_id = azuread_application.current.application_id app_role_assignment_required = false owners = var.application_owners @@ -16,12 +14,9 @@ resource "azuread_service_principal" "current" { } resource "azuread_service_principal_password" "current" { - service_principal_id = azuread_service_principal.current.id } - - resource "azuread_application_federated_identity_credential" "example" { application_object_id = azuread_application.current.object_id display_name = var.application_display_name diff --git a/terraform/azure/service-principal/providers.tf b/terraform/azure/service-principal/providers.tf deleted file mode 100644 index a3e5ab724..000000000 --- a/terraform/azure/service-principal/providers.tf +++ /dev/null @@ -1,11 +0,0 @@ -terraform { - required_providers { - azuread = { - source = "hashicorp/azuread" - } - } -} - -provider "azurerm" { - features {} -} diff --git a/terraform/azure/workload-identity/main.tf b/terraform/azure/workload-identity/main.tf index 459d3b671..fd7a638f0 100644 --- a/terraform/azure/workload-identity/main.tf +++ b/terraform/azure/workload-identity/main.tf @@ -13,7 +13,7 @@ resource "helm_release" "azure-workload-identity-system" { namespace = "azure-workload-identity-system" chart = "workload-identity-webhook" repository = "https://azure.github.io/azure-workload-identity/charts" - wait = false + wait = true depends_on = [kubernetes_namespace.azure-workload-identity-system] set { diff --git a/terraform/azure/workload-identity/provider.tf b/terraform/azure/workload-identity/provider.tf deleted file mode 100644 index 9fccea60d..000000000 --- a/terraform/azure/workload-identity/provider.tf +++ /dev/null @@ -1,5 +0,0 @@ -provider "helm" { - kubernetes { - config_path = "~/.kube/config" - } -}