feat: add AKS e2e managed (#2811)

Migrate azure e2e tests to use the new TFC_* secrets which are
provisioned through external-secrets/infrastructure.
Also enable the use of `/ok-to-test-managed provider=azure` command
to run e2e managed tests that verify integration with AKS and
Azure Workload Identity (AZWI).

Signed-off-by: Moritz Johner <beller.moritz@googlemail.com>

.github/actions/e2e-managed/action.yml

@@ -37,12 +37,6 @@ runs:
           });
           return result;
 
-    - name: Configure AWS Credentials
-      uses: aws-actions/configure-aws-credentials@v1
-      with:
-        role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }}
-        aws-region: ${{ env.AWS_REGION }}
-
     - name: Setup Go
       uses: actions/setup-go@v3
       with:
@@ -78,6 +72,13 @@ runs:
       shell: bash
       run: find ${{ github.workspace }} | grep tf$ | xargs -n1 dirname | xargs -IXXX -n1 /bin/sh -c 'set -o errexit; cd XXX; pwd; tflint --loglevel=info .; cd - >/dev/null'
 
+    - name: Configure AWS Credentials
+      if: env.CLOUD_PROVIDER == 'aws'
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }}
+        aws-region: ${{ env.AWS_REGION }}
+
     - name: Setup TF Gcloud Provider
       shell: bash
       if: env.CLOUD_PROVIDER == 'gcp'
@@ -87,8 +88,20 @@ runs:
         mkdir -p terraform/gcp/secrets
         echo ${GCP_SM_SA_GKE_JSON} > terraform/gcp/secrets/gcloud-service-account-key.json
 
+    - name: 'Az CLI login'
+      uses: azure/login@v1
+      if: env.CLOUD_PROVIDER == 'azure'
+      with:
+        client-id: ${{ env.TFC_AZURE_CLIENT_ID }}
+        tenant-id: ${{ env.TFC_AZURE_TENANT_ID }}
+        subscription-id: ${{ env.TFC_AZURE_SUBSCRIPTION_ID }}
+
     - name: Show TF
       shell: bash
+      env:
+        ARM_CLIENT_ID: "${{ env.TFC_AZURE_CLIENT_ID }}"
+        ARM_SUBSCRIPTION_ID: "${{ env.TFC_AZURE_SUBSCRIPTION_ID }}"
+        ARM_TENANT_ID: "${{ env.TFC_AZURE_TENANT_ID }}"
       run: |-
         PROVIDER=${{env.CLOUD_PROVIDER}}
         make tf.show.${PROVIDER}
@@ -96,7 +109,9 @@ runs:
     - name: Apply TF
       shell: bash
       env:
-        TF_VAR_OIDC_TOKEN: "${{steps.fetch-token.outputs.result}}"
+        ARM_CLIENT_ID: "${{ env.TFC_AZURE_CLIENT_ID }}"
+        ARM_SUBSCRIPTION_ID: "${{ env.TFC_AZURE_SUBSCRIPTION_ID }}"
+        ARM_TENANT_ID: "${{ env.TFC_AZURE_TENANT_ID }}"
       run: |-
         PROVIDER=${{env.CLOUD_PROVIDER}}
         make tf.apply.${PROVIDER}
@@ -121,6 +136,12 @@ runs:
       run: |-
         aws --region $AWS_REGION eks update-kubeconfig --name $AWS_CLUSTER_NAME
     
+    - name: Get AKS credentials
+      if: env.CLOUD_PROVIDER == 'azure'
+      shell: bash
+      run: |-
+        az aks get-credentials --admin --name eso-cluster --resource-group external-secrets-operator
+
     - name: Login to Docker
       uses: docker/login-action@v2
       if: env.GHCR_USERNAME != ''
@@ -137,11 +158,15 @@ runs:
         export PATH=$PATH:$(go env GOPATH)/bin
         PROVIDER=${{env.CLOUD_PROVIDER}}
         go install github.com/onsi/ginkgo/v2/ginkgo@v2.1.6
-        make test.e2e.managed GINKGO_LABELS="${PROVIDER}" TEST_SUITES="provider"
+        make test.e2e.managed GINKGO_LABELS="${PROVIDER} && managed" TEST_SUITES="provider"
 
     - name: Destroy TF
       shell: bash
       if: always()
+      env:
+        ARM_CLIENT_ID: "${{ env.TFC_AZURE_CLIENT_ID }}"
+        ARM_SUBSCRIPTION_ID: "${{ env.TFC_AZURE_SUBSCRIPTION_ID }}"
+        ARM_TENANT_ID: "${{ env.TFC_AZURE_TENANT_ID }}"
       run: |-
         PROVIDER=${{env.CLOUD_PROVIDER}}
         make tf.destroy.${PROVIDER}

.github/workflows/e2e-managed.yml

@@ -39,10 +39,11 @@ env:
   TF_VAR_AWS_REGION: "eu-central-1"
   TF_VAR_AWS_CLUSTER_NAME: "eso-e2e-managed"
 
-  AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID}}
+  TFC_AZURE_CLIENT_ID: ${{ secrets.TFC_AZURE_CLIENT_ID}}
-  AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET}}
+  TFC_AZURE_CLIENT_SECRET: ${{ secrets.TFC_AZURE_CLIENT_SECRET }}
-  TENANT_ID: ${{ secrets.TENANT_ID}}
+  TFC_AZURE_TENANT_ID: ${{ secrets.TFC_AZURE_TENANT_ID}}
-  VAULT_URL: ${{ secrets.VAULT_URL}}
+  TFC_AZURE_SUBSCRIPTION_ID: ${{ secrets.TFC_AZURE_SUBSCRIPTION_ID }}
+  TFC_VAULT_URL: ${{ secrets.TFC_VAULT_URL}}
 
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   GITHUB_PR_NUMBER: ${{ github.event.client_payload.pull_request.number }}
@@ -66,7 +67,6 @@ jobs:
 
     - uses: ./.github/actions/e2e-managed
       env:
-        CLOUD_PROVIDER: aws
         GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
 
   integration-managed:

.github/workflows/e2e.yml

@@ -32,10 +32,12 @@ env:
   AWS_REGION: "eu-central-1"
   AWS_OIDC_ROLE_ARN: ${{ secrets.AWS_OIDC_ROLE_ARN }}
 
-  AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID}}
+  TFC_AZURE_CLIENT_ID: ${{ secrets.TFC_AZURE_CLIENT_ID}}
-  AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET}}
+  TFC_AZURE_CLIENT_SECRET: ${{ secrets.TFC_AZURE_CLIENT_SECRET }}
-  TENANT_ID: ${{ secrets.TENANT_ID}}
+  TFC_AZURE_TENANT_ID: ${{ secrets.TFC_AZURE_TENANT_ID}}
-  VAULT_URL: ${{ secrets.VAULT_URL}}
+  TFC_AZURE_SUBSCRIPTION_ID: ${{ secrets.TFC_AZURE_SUBSCRIPTION_ID }}
+  TFC_VAULT_URL: ${{ secrets.TFC_VAULT_URL}}
+  
   SCALEWAY_API_URL: ${{ secrets.SCALEWAY_API_URL }}
   SCALEWAY_REGION: ${{ secrets.SCALEWAY_REGION }}
   SCALEWAY_PROJECT_ID: ${{ secrets.SCALEWAY_PROJECT_ID }}

e2e/go.mod

@@ -40,6 +40,7 @@ replace (
 require (
 	cloud.google.com/go/secretmanager v1.11.2
 	github.com/Azure/azure-sdk-for-go v68.0.0+incompatible
+	github.com/Azure/go-autorest/autorest v0.11.29
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.12
 	github.com/DelineaXPM/dsv-sdk-go/v2 v2.1.0
 	github.com/akeylesslabs/akeyless-go-cloud-id v0.3.4
@@ -74,7 +75,6 @@ require (
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v1.1.3 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest v0.11.29 // indirect
 	github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect
 	github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
@@ -82,6 +82,7 @@ require (
 	github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
@@ -105,6 +106,7 @@ require (
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
@@ -130,6 +132,7 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect
 	github.com/lestrrat-go/blackmagic v1.0.2 // indirect
 	github.com/lestrrat-go/httpcc v1.0.1 // indirect

e2e/go.sum

@@ -76,6 +76,8 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z
 github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 h1:hVeq+yCyUi+MsoO/CU95yqCIcdzra5ovzk8Q2BBpV2M=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DelineaXPM/dsv-sdk-go/v2 v2.1.0 h1:+XXJ43iH4js8LIBr4MUGq1J09ycivNkTNhtn4mFyhY8=
@@ -186,6 +188,8 @@ github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw
 github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
 github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE=
+github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -332,6 +336,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/lestrrat-go/backoff/v2 v2.0.8 h1:oNb5E5isby2kiro9AgdHLv5N5tint1AnDVVf2E2un5A=
 github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y=
 github.com/lestrrat-go/blackmagic v1.0.1/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=

e2e/run.sh

@@ -44,7 +44,7 @@ kubectl run --rm \
   --attach \
   --restart=Never \
   --pod-running-timeout=5m \
-  --labels="app=eso-e2e" \
+  --labels="app=eso-e2e,azure.workload.identity/use=true" \
   --env="ACK_GINKGO_DEPRECATIONS=2.9.5" \
   --env="GINKGO_LABELS=${GINKGO_LABELS:-.*}" \
   --env="GCP_SM_SA_JSON=${GCP_SM_SA_JSON:-}" \
@@ -59,13 +59,13 @@ kubectl run --rm \
   --env="AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-}" \
   --env="AWS_SA_NAME=${AWS_SA_NAME:-}" \
   --env="AWS_SA_NAMESPACE=${AWS_SA_NAMESPACE:-}" \
-  --env="AZURE_CLIENT_ID=${AZURE_CLIENT_ID:-}" \
+  --env="TFC_AZURE_CLIENT_ID=${TFC_AZURE_CLIENT_ID:-}" \
-  --env="AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET:-}" \
+  --env="TFC_AZURE_CLIENT_SECRET=${TFC_AZURE_CLIENT_SECRET:-}" \
+  --env="TFC_AZURE_TENANT_ID=${TFC_AZURE_TENANT_ID:-}" \
+  --env="TFC_VAULT_URL=${TFC_VAULT_URL:-}" \
   --env="AKEYLESS_ACCESS_ID=${AKEYLESS_ACCESS_ID:-}" \
   --env="AKEYLESS_ACCESS_TYPE=${AKEYLESS_ACCESS_TYPE:-}" \
   --env="AKEYLESS_ACCESS_TYPE_PARAM=${AKEYLESS_ACCESS_TYPE_PARAM:-}" \
-  --env="TENANT_ID=${TENANT_ID:-}" \
-  --env="VAULT_URL=${VAULT_URL:-}" \
   --env="GITLAB_TOKEN=${GITLAB_TOKEN:-}" \
   --env="GITLAB_PROJECT_ID=${GITLAB_PROJECT_ID:-}" \
   --env="GITLAB_ENVIRONMENT=${GITLAB_ENVIRONMENT:-}" \

e2e/suites/provider/cases/azure/azure_managed.go

@@ -34,15 +34,15 @@ const (
 // to test workload-identity authentication.
 var _ = Describe("[azuremanaged] with pod identity", Label("azure", "keyvault", "managed", "workload-identity"), func() {
 	f := framework.New("eso-azuremanaged")
-	prov := newFromEnv(f)
+	prov := newFromWorkloadIdentity(f)
 
 	// each test case gets its own ESO instance
 	BeforeEach(func() {
 		f.Install(addon.NewESO(
 			addon.WithControllerClass(f.BaseName),
-			addon.WithServiceAccount(prov.clientID),
 			addon.WithReleaseName(f.Namespace.Name),
-			addon.WithNamespace("default"),
+			addon.WithNamespace("external-secrets-operator"),
+			addon.WithServiceAccount("external-secrets-operator"),
 			addon.WithoutWebhook(),
 			addon.WithoutCertController(),
 		))

e2e/suites/provider/cases/azure/provider.go

@@ -15,10 +15,13 @@ package azure
 import (
 	"context"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/profiles/latest/keyvault/keyvault"
+	"github.com/Azure/go-autorest/autorest"
+	"github.com/Azure/go-autorest/autorest/azure"
 	kvauth "github.com/Azure/go-autorest/autorest/azure/auth"
 
 	// nolint
@@ -32,6 +35,7 @@ import (
 	"github.com/external-secrets/external-secrets-e2e/framework"
 	esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
 	esmeta "github.com/external-secrets/external-secrets/apis/meta/v1"
+	esoazkv "github.com/external-secrets/external-secrets/pkg/provider/azure/keyvault"
 )
 
 type azureProvider struct {
@@ -43,30 +47,38 @@ type azureProvider struct {
 	framework    *framework.Framework
 }
 
-func newazureProvider(f *framework.Framework, clientID, clientSecret, tenantID, vaultURL string) *azureProvider {
+// newFromEnv creates a new Azure KeyVault e2e test provider
-	clientCredentialsConfig := kvauth.NewClientCredentialsConfig(clientID, clientSecret, tenantID)
+// which uses client credentials flow to authenticate with azure.
-	clientCredentialsConfig.Resource = "https://vault.azure.net"
+func newFromEnv(f *framework.Framework) *azureProvider {
+	vaultURL := os.Getenv("TFC_VAULT_URL")
+	tenantID := os.Getenv("TFC_AZURE_TENANT_ID")
+	clientID := os.Getenv("TFC_AZURE_CLIENT_ID")
+	clientSecret := os.Getenv("TFC_AZURE_CLIENT_SECRET")
+
 	basicClient := keyvault.New()
 	prov := &azureProvider{
 		framework:    f,
-		client:       &basicClient,
 		clientID:     clientID,
-		clientSecret: clientSecret,
 		tenantID:     tenantID,
 		vaultURL:     vaultURL,
+		client:       &basicClient,
+		clientSecret: clientSecret,
 	}
 
 	o := &sync.Once{}
 	BeforeEach(func() {
 		// run authorizor only if this spec is called
+		// this allows us to run OTHER providers using GINKGO_LABELS without bailing out
 		o.Do(func() {
+			defer GinkgoRecover()
+			clientCredentialsConfig := kvauth.NewClientCredentialsConfig(clientID, clientSecret, tenantID)
+			clientCredentialsConfig.Resource = "https://vault.azure.net"
 			authorizer, err := clientCredentialsConfig.Authorizer()
 			if err != nil {
 				Fail(err.Error())
 			}
 			prov.client.Authorizer = authorizer
 		})
-		prov.CreateSecretStoreWithWI()
 		prov.CreateSecretStore()
 		prov.CreateReferentSecretStore()
 	})
@@ -74,12 +86,51 @@ func newazureProvider(f *framework.Framework, clientID, clientSecret, tenantID,
 	return prov
 }
 
-func newFromEnv(f *framework.Framework) *azureProvider {
+// create a new provider from workload identity
-	vaultURL := os.Getenv("VAULT_URL")
+// the azwi webhook injects `AZURE_*` env vars into the container.
-	tenantID := os.Getenv("TENANT_ID")
+// we use these credentials to authenticate with azure using the federated token flow.
+// please see here for details: https://azure.github.io/azure-workload-identity/docs/quick-start.html
+func newFromWorkloadIdentity(f *framework.Framework) *azureProvider {
+	// from azwi webhook
+	tenantID := os.Getenv("AZURE_TENANT_ID")
 	clientID := os.Getenv("AZURE_CLIENT_ID")
-	clientSecret := os.Getenv("AZURE_CLIENT_SECRET")
+	tokenFilePath := os.Getenv("AZURE_FEDERATED_TOKEN_FILE")
-	return newazureProvider(f, clientID, clientSecret, tenantID, vaultURL)
+
+	// from run.sh
+	vaultURL := "https://eso-testing.vault.azure.net/"
+
+	basicClient := keyvault.New()
+	prov := &azureProvider{
+		framework: f,
+		client:    &basicClient,
+		clientID:  clientID,
+		tenantID:  tenantID,
+		vaultURL:  vaultURL,
+	}
+
+	o := &sync.Once{}
+	BeforeEach(func() {
+		prov.CreateSecretStoreWithWI()
+		// run authorizor only if this spec is called
+		o.Do(func() {
+			defer GinkgoRecover()
+			token, err := os.ReadFile(tokenFilePath)
+			if err != nil {
+				Fail(err.Error())
+			}
+
+			// exchange the federated token for an access token
+			aadEndpoint := esoazkv.AadEndpointForType(esv1beta1.AzureEnvironmentPublicCloud)
+			kvResource := strings.TrimSuffix(azure.PublicCloud.KeyVaultEndpoint, "/")
+			tokenProvider, err := esoazkv.NewTokenProvider(context.Background(), string(token), clientID, tenantID, aadEndpoint, kvResource)
+			if err != nil {
+				Fail(err.Error())
+			}
+			basicClient.Authorizer = autorest.NewBearerAuthorizer(tokenProvider)
+		})
+	})
+
+	return prov
 }
 
 func (s *azureProvider) CreateSecret(key string, val framework.SecretEntry) {

terraform/azure/aks/providers.tf

@@ -1,13 +0,0 @@
-terraform {
-  required_providers {
-    azurerm = {
-      source = "hashicorp/azurerm"
-    }
-  }
-}
-
-provider "azurerm" {
-  features {}
-}
-
-

terraform/azure/key-vault/main.tf

@@ -16,6 +16,12 @@ resource "azurerm_key_vault" "current" {
 
     key_permissions = [
       "Get",
+      "List",
+      "Create",
+      "Delete",
+      "Purge",
+      "Decrypt",
+      "Encrypt",
     ]
 
     secret_permissions = [
@@ -27,7 +33,11 @@ resource "azurerm_key_vault" "current" {
     ]
 
     storage_permissions = [
+      "Set",
       "Get",
+      "Delete",
+      "Purge",
+      "Recover"
     ]
   }
   access_policy {
@@ -36,7 +46,42 @@ resource "azurerm_key_vault" "current" {
 
     secret_permissions = [
       "Get",
+      "Set",
+      "Delete",
+      "Purge",
+      "Recover",
     ]
 
   }
+
+  access_policy {
+    tenant_id = var.tenant_id
+    object_id = var.eso_e2e_sp_object_id
+
+    secret_permissions = [
+      "Get",
+      "Set",
+      "Delete",
+      "Purge",
+      "Recover",
+    ]
+
+    key_permissions = [
+      "Get",
+      "List",
+      "Create",
+      "Delete",
+      "Purge",
+      "Decrypt",
+      "Encrypt",
+    ]
+
+    certificate_permissions = [
+      "Get",
+      "List",
+      "Create",
+      "Delete",
+      "Purge",
+    ]
+  }
 }

terraform/azure/key-vault/providers.tf

@@ -1,11 +0,0 @@
-terraform {
-  required_providers {
-    azurerm = {
-      source = "hashicorp/azurerm"
-    }
-  }
-}
-
-provider "azurerm" {
-  features {}
-}

terraform/azure/key-vault/variables.tf

@@ -22,3 +22,8 @@ variable "eso_sp_object_id" {
   type        = string
   description = "The object ID of the ESO service account"
 }
+
+variable "eso_e2e_sp_object_id" {
+  type        = string
+  description = "The object ID of the ESO e2e service account"
+}

terraform/azure/main.tf

@@ -2,11 +2,9 @@ data "azurerm_client_config" "current" {}
 
 data "azurerm_subscription" "primary" {}
 
-module "test_resource_group" {
+resource "azurerm_resource_group" "current" {
-  source = "./resource-group"
+  name     = var.resource_group_name
-
+  location = var.resource_group_location
-  resource_group_name     = var.resource_group_name
-  resource_group_location = var.resource_group_location
 }
 
 module "test_sp" {
@@ -16,6 +14,19 @@ module "test_sp" {
   application_owners       = [data.azurerm_client_config.current.object_id]
   issuer                   = module.test_aks.cluster_issuer_url
   subject                  = "system:serviceaccount:${var.sa_namespace}:${var.sa_name}"
+
+  depends_on = [
+    azurerm_resource_group.current
+  ]
+}
+
+module "e2e_sp" {
+  source = "./service-principal"
+
+  application_display_name = var.application_display_name
+  application_owners       = [data.azurerm_client_config.current.object_id]
+  issuer                   = module.test_aks.cluster_issuer_url
+  subject                  = "system:serviceaccount:default:external-secrets-e2e"
 }
 
 module "test_key_vault" {
@@ -27,6 +38,11 @@ module "test_key_vault" {
   tenant_id               = data.azurerm_client_config.current.tenant_id
   client_object_id        = data.azurerm_client_config.current.object_id
   eso_sp_object_id        = module.test_sp.sp_object_id
+  eso_e2e_sp_object_id    = module.e2e_sp.sp_object_id
+
+  depends_on = [
+    azurerm_resource_group.current
+  ]
 }
 
 module "test_workload_identity" {
@@ -46,12 +62,43 @@ module "test_aks" {
   default_node_pool_node_count = var.default_node_pool_node_count
   default_node_pool_vm_size    = var.default_node_pool_vm_size
   cluster_tags                 = var.cluster_tags
+
+  depends_on = [
+    azurerm_resource_group.current
+  ]
 }
 
 resource "azurerm_role_assignment" "current" {
   scope                = data.azurerm_subscription.primary.id
-  role_definition_name = "Reader"
+  role_definition_name = "Owner"
   principal_id         = module.test_sp.sp_id
+
+  depends_on = [
+    azurerm_resource_group.current
+  ]
+}
+
+resource "kubernetes_namespace" "eso" {
+  metadata {
+    name = "external-secrets-operator"
+  }
+}
+
+// the `e2e` pod itself runs with workload identity and
+// does not rely on client credentials.
+resource "kubernetes_service_account" "e2e" {
+  metadata {
+    name      = "external-secrets-e2e"
+    namespace = "default"
+    annotations = {
+      "azure.workload.identity/client-id" = module.e2e_sp.application_id
+      "azure.workload.identity/tenant-id" = data.azurerm_client_config.current.tenant_id
+    }
+    labels = {
+      "azure.workload.identity/use" = "true"
+    }
+  }
+  depends_on = [module.test_aks, kubernetes_namespace.eso]
 }
 
 resource "kubernetes_service_account" "current" {
@@ -66,5 +113,5 @@ resource "kubernetes_service_account" "current" {
       "azure.workload.identity/use" = "true"
     }
   }
-
+  depends_on = [module.test_aks, kubernetes_namespace.eso]
 }

terraform/azure/providers.tf

@@ -8,14 +8,28 @@ terraform {
 
 provider "azurerm" {
   features {}
+  # set this to false when running locally
+  use_oidc = true 
+}
+
+data "azurerm_kubernetes_cluster" "default" {
+  depends_on          = [module.test_aks] # refresh cluster state before reading
+  name                = var.cluster_name
+  resource_group_name = var.resource_group_name
 }
 
 provider "helm" {
   kubernetes {
-    config_path = "~/.kube/config"
+    host                   = data.azurerm_kubernetes_cluster.default.kube_config.0.host
+    client_certificate     = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.client_certificate)
+    client_key             = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.client_key)
+    cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.cluster_ca_certificate)
   }
 }
-provider "kubernetes" {
-  config_path = "~/.kube/config"
-}
 
+provider "kubernetes" {
+  host                   = data.azurerm_kubernetes_cluster.default.kube_config.0.host
+  client_certificate     = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.client_certificate)
+  client_key             = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.client_key)
+  cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.cluster_ca_certificate)
+}

terraform/azure/resource-group/main.tf

@@ -1,4 +0,0 @@
-resource "azurerm_resource_group" "current" {
-  name     = var.resource_group_name
-  location = var.resource_group_location
-}

terraform/azure/resource-group/providers.tf

@@ -1,11 +0,0 @@
-terraform {
-  required_providers {
-    azurerm = {
-      source = "hashicorp/azurerm"
-    }
-  }
-}
-
-provider "azurerm" {
-  features {}
-}

terraform/azure/resource-group/variables.tf

@@ -1,9 +0,0 @@
-variable "resource_group_name" {
-  type        = string
-  description = "The Name which should be used for this Resource Group"
-}
-
-variable "resource_group_location" {
-  type        = string
-  description = "The Azure Region where the Resource Group should exist"
-}

terraform/azure/service-principal/main.tf

@@ -1,11 +1,9 @@
 resource "azuread_application" "current" {
-
   display_name = var.application_display_name
   owners       = var.application_owners
 }
 
 resource "azuread_service_principal" "current" {
-
   application_id               = azuread_application.current.application_id
   app_role_assignment_required = false
   owners                       = var.application_owners
@@ -16,12 +14,9 @@ resource "azuread_service_principal" "current" {
 }
 
 resource "azuread_service_principal_password" "current" {
-
   service_principal_id = azuread_service_principal.current.id
 }
 
-
-
 resource "azuread_application_federated_identity_credential" "example" {
   application_object_id = azuread_application.current.object_id
   display_name          = var.application_display_name

terraform/azure/service-principal/providers.tf

@@ -1,11 +0,0 @@
-terraform {
-  required_providers {
-    azuread = {
-      source = "hashicorp/azuread"
-    }
-  }
-}
-
-provider "azurerm" {
-  features {}
-}

terraform/azure/workload-identity/main.tf

@@ -13,7 +13,7 @@ resource "helm_release" "azure-workload-identity-system" {
   namespace  = "azure-workload-identity-system"
   chart      = "workload-identity-webhook"
   repository = "https://azure.github.io/azure-workload-identity/charts"
-  wait       = false
+  wait       = true
   depends_on = [kubernetes_namespace.azure-workload-identity-system]
 
   set {

terraform/azure/workload-identity/provider.tf

@@ -1,5 +0,0 @@
-provider "helm" {
-  kubernetes {
-    config_path = "~/.kube/config"
-  }
-}