[Feature] Check if Volume with LocalStorage is missing (#1315)

CHANGELOG.md

@@ -9,6 +9,7 @@
 - (Feature) Features startup logging
 - (Maintenance) Generics for type handling
 - (Bugfix) Fix creating sync components with EA type set to Managed and headless svc
+- (Feature) Check if Volume with LocalStorage is missing
 
 ## [1.2.27](https://github.com/arangodb/kube-arangodb/tree/1.2.27) (2023-04-27)
 - (Feature) Add InSync Cache

pkg/deployment/reconcile/plan_builder_volume.go

@@ -26,6 +26,7 @@ import (
 	core "k8s.io/api/core/v1"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	sharedApis "github.com/arangodb/kube-arangodb/pkg/apis/shared"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/reconcile/shared"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 	inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector"
@@ -51,7 +52,8 @@ func (r *Reconciler) updateMemberConditionTypeMemberVolumeUnschedulableCondition
 				if volumeName := pvc.Spec.VolumeName; volumeName != "" {
 					if pv, ok := volumeClient.GetSimple(volumeName); ok {
 						// We have volume and volumeclaim, lets calculate condition
-						unschedulable := memberConditionTypeMemberVolumeUnschedulableCalculate(cache, pv, pvc)
+						unschedulable := memberConditionTypeMemberVolumeUnschedulableCalculate(cache, pv, pvc,
+							memberConditionTypeMemberVolumeUnschedulableLocalStorageGone)
 
 						if unschedulable == e.Member.Conditions.IsTrue(api.ConditionTypeMemberVolumeUnschedulable) {
 							continue
@@ -82,3 +84,37 @@ func memberConditionTypeMemberVolumeUnschedulableCalculate(cache inspectorInterf
 
 	return false
 }
+
+func memberConditionTypeMemberVolumeUnschedulableLocalStorageGone(cache inspectorInterface.Inspector, pv *core.PersistentVolume, _ *core.PersistentVolumeClaim) bool {
+	nodes, err := cache.Node().V1()
+	if err != nil {
+		return false
+	}
+
+	if pv.Spec.PersistentVolumeSource.Local == nil {
+		// We are not on LocalStorage
+		return false
+	}
+
+	if nodeAffinity := pv.Spec.NodeAffinity; nodeAffinity != nil {
+		if required := nodeAffinity.Required; required != nil {
+			for _, nst := range required.NodeSelectorTerms {
+				for _, expr := range nst.MatchExpressions {
+					if expr.Key == sharedApis.TopologyKeyHostname && expr.Operator == core.NodeSelectorOpIn {
+						// We got exact key which is required for PV
+						if len(expr.Values) == 1 {
+							// Only one host assigned, we use it as localStorage - check if node exists
+							_, ok := nodes.GetSimple(expr.Values[0])
+							if !ok {
+								// Node is missing!
+								return true
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return false
+}

pkg/deployment/reconcile/plan_builder_volume_test.go

@@ -0,0 +1,263 @@
+//
+// DISCLAIMER
+//
+// Copyright 2023 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package reconcile
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	core "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/arangodb/kube-arangodb/pkg/apis/shared"
+	"github.com/arangodb/kube-arangodb/pkg/util/kclient"
+	"github.com/arangodb/kube-arangodb/pkg/util/tests"
+)
+
+func Test_MemberConditionTypeMemberVolumeUnschedulableLocalStorageGone(t *testing.T) {
+	type testCase struct {
+		pv core.PersistentVolumeSpec
+
+		node *core.Node
+
+		result bool
+	}
+
+	testCases := map[string]testCase{
+		"Non LocalVolume": {
+			pv: core.PersistentVolumeSpec{
+				PersistentVolumeSource: core.PersistentVolumeSource{
+					GCEPersistentDisk: &core.GCEPersistentDiskVolumeSource{},
+				},
+				NodeAffinity: nil,
+			},
+		},
+		"LocalVolume without selectors": {
+			pv: core.PersistentVolumeSpec{
+				PersistentVolumeSource: core.PersistentVolumeSource{
+					Local: &core.LocalVolumeSource{},
+				},
+				NodeAffinity: nil,
+			},
+		},
+		"LocalVolume with partial selectors - NPE#1": {
+			pv: core.PersistentVolumeSpec{
+				PersistentVolumeSource: core.PersistentVolumeSource{
+					Local: &core.LocalVolumeSource{},
+				},
+				NodeAffinity: &core.VolumeNodeAffinity{},
+			},
+		},
+		"LocalVolume with partial selectors - NPE#2": {
+			pv: core.PersistentVolumeSpec{
+				PersistentVolumeSource: core.PersistentVolumeSource{
+					Local: &core.LocalVolumeSource{},
+				},
+				NodeAffinity: &core.VolumeNodeAffinity{
+					Required: &core.NodeSelector{},
+				},
+			},
+		},
+		"LocalVolume with partial selectors - NPE#3": {
+			pv: core.PersistentVolumeSpec{
+				PersistentVolumeSource: core.PersistentVolumeSource{
+					Local: &core.LocalVolumeSource{},
+				},
+				NodeAffinity: &core.VolumeNodeAffinity{
+					Required: &core.NodeSelector{
+						NodeSelectorTerms: []core.NodeSelectorTerm{},
+					},
+				},
+			},
+		},
+		"LocalVolume with partial selectors - NPE#4": {
+			pv: core.PersistentVolumeSpec{
+				PersistentVolumeSource: core.PersistentVolumeSource{
+					Local: &core.LocalVolumeSource{},
+				},
+				NodeAffinity: &core.VolumeNodeAffinity{
+					Required: &core.NodeSelector{
+						NodeSelectorTerms: []core.NodeSelectorTerm{
+							{
+								MatchExpressions: []core.NodeSelectorRequirement{},
+							},
+						},
+					},
+				},
+			},
+		},
+		"LocalVolume with invalid selector key": {
+			pv: core.PersistentVolumeSpec{
+				PersistentVolumeSource: core.PersistentVolumeSource{
+					Local: &core.LocalVolumeSource{},
+				},
+				NodeAffinity: &core.VolumeNodeAffinity{
+					Required: &core.NodeSelector{
+						NodeSelectorTerms: []core.NodeSelectorTerm{
+							{
+								MatchExpressions: []core.NodeSelectorRequirement{
+									{
+										Key:      shared.NodeArchAffinityLabel,
+										Operator: core.NodeSelectorOpIn,
+										Values: []string{
+											"node",
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		"LocalVolume with invalid selector operator": {
+			pv: core.PersistentVolumeSpec{
+				PersistentVolumeSource: core.PersistentVolumeSource{
+					Local: &core.LocalVolumeSource{},
+				},
+				NodeAffinity: &core.VolumeNodeAffinity{
+					Required: &core.NodeSelector{
+						NodeSelectorTerms: []core.NodeSelectorTerm{
+							{
+								MatchExpressions: []core.NodeSelectorRequirement{
+									{
+										Key:      shared.TopologyKeyHostname,
+										Operator: core.NodeSelectorOpDoesNotExist,
+										Values: []string{
+											"node",
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		"LocalVolume with valid selector - existing node": {
+			pv: core.PersistentVolumeSpec{
+				PersistentVolumeSource: core.PersistentVolumeSource{
+					Local: &core.LocalVolumeSource{},
+				},
+				NodeAffinity: &core.VolumeNodeAffinity{
+					Required: &core.NodeSelector{
+						NodeSelectorTerms: []core.NodeSelectorTerm{
+							{
+								MatchExpressions: []core.NodeSelectorRequirement{
+									{
+										Key:      shared.TopologyKeyHostname,
+										Operator: core.NodeSelectorOpIn,
+										Values: []string{
+											"node",
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+
+			node: &core.Node{
+				ObjectMeta: meta.ObjectMeta{
+					Name: "node",
+				},
+			},
+		},
+		"LocalVolume with valid selector - missing node #1": {
+			pv: core.PersistentVolumeSpec{
+				PersistentVolumeSource: core.PersistentVolumeSource{
+					Local: &core.LocalVolumeSource{},
+				},
+				NodeAffinity: &core.VolumeNodeAffinity{
+					Required: &core.NodeSelector{
+						NodeSelectorTerms: []core.NodeSelectorTerm{
+							{
+								MatchExpressions: []core.NodeSelectorRequirement{
+									{
+										Key:      shared.TopologyKeyHostname,
+										Operator: core.NodeSelectorOpIn,
+										Values: []string{
+											"node",
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+
+			node: &core.Node{
+				ObjectMeta: meta.ObjectMeta{
+					Name: "node1",
+				},
+			},
+
+			result: true,
+		},
+		"LocalVolume with valid selector - missing node #2": {
+			pv: core.PersistentVolumeSpec{
+				PersistentVolumeSource: core.PersistentVolumeSource{
+					Local: &core.LocalVolumeSource{},
+				},
+				NodeAffinity: &core.VolumeNodeAffinity{
+					Required: &core.NodeSelector{
+						NodeSelectorTerms: []core.NodeSelectorTerm{
+							{
+								MatchExpressions: []core.NodeSelectorRequirement{
+									{
+										Key:      shared.TopologyKeyHostname,
+										Operator: core.NodeSelectorOpIn,
+										Values: []string{
+											"node",
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+
+			result: true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			client := kclient.NewFakeClient()
+
+			if tc.node != nil {
+				_, err := client.Kubernetes().CoreV1().Nodes().Create(context.Background(), tc.node, meta.CreateOptions{})
+				require.NoError(t, err)
+			}
+
+			ins := tests.NewInspector(t, client)
+
+			require.Equal(t, tc.result, memberConditionTypeMemberVolumeUnschedulableLocalStorageGone(ins, &core.PersistentVolume{
+				Spec: tc.pv,
+			}, &core.PersistentVolumeClaim{}))
+		})
+	}
+
+}

pkg/storage/pv_creator.go

@@ -37,6 +37,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"github.com/arangodb/kube-arangodb/pkg/apis/shared"
 	api "github.com/arangodb/kube-arangodb/pkg/apis/storage/v1alpha"
 	"github.com/arangodb/kube-arangodb/pkg/storage/provisioner"
 	resources "github.com/arangodb/kube-arangodb/pkg/storage/resources"
@@ -271,7 +272,7 @@ func createNodeSelector(nodeName string) *core.NodeSelector {
 			core.NodeSelectorTerm{
 				MatchExpressions: []core.NodeSelectorRequirement{
 					core.NodeSelectorRequirement{
-						Key:      "kubernetes.io/hostname",
+						Key:      shared.TopologyKeyHostname,
 						Operator: core.NodeSelectorOpIn,
 						Values:   []string{nodeName},
 					},