Inspect pods, update Ready condition and recreate pod when deleted

pkg/controller/controller.go

@@ -230,7 +230,7 @@ func (c *Controller) handleDeploymentEvent(event *Event) error {
 			return maskAny(fmt.Errorf("unsafe state. deployment (%s) was created before but we received event (%s)", apiObject.Name, event.Type))
 		}
 
-		cfg, deps := c.makeDeploymentConfigAndDeps()
+		cfg, deps := c.makeDeploymentConfigAndDeps(apiObject)
 		nc, err := deployment.New(cfg, deps, apiObject)
 		if err != nil {
 			return maskAny(fmt.Errorf("failed to create deployment: %s", err))
@@ -262,12 +262,15 @@ func (c *Controller) handleDeploymentEvent(event *Event) error {
 }
 
 // makeDeploymentConfigAndDeps creates a Config & Dependencies object for a new cluster.
-func (c *Controller) makeDeploymentConfigAndDeps() (deployment.Config, deployment.Dependencies) {
+func (c *Controller) makeDeploymentConfigAndDeps(apiObject *api.ArangoDeployment) (deployment.Config, deployment.Dependencies) {
 	cfg := deployment.Config{
 		ServiceAccount: c.Config.ServiceAccount,
 	}
 	deps := deployment.Dependencies{
-		Log:           c.Dependencies.Log,
+		Log: c.Dependencies.Log.With().
+			Str("component", "deployment").
+			Str("deployment", apiObject.GetName()).
+			Logger(),
 		KubeCli:       c.Dependencies.KubeCli,
 		DatabaseCRCli: c.Dependencies.DatabaseCRCli,
 	}

pkg/deployment/deployment.go

@@ -204,6 +204,10 @@ func (d *Deployment) run() {
 			if err := d.inspectPods(); err != nil {
 				d.createEvent(k8sutil.NewErrorEvent("Pod inspection failed", err, d.apiObject))
 			}
+			// Ensure all resources are created
+			if err := d.ensurePods(d.apiObject); err != nil {
+				d.createEvent(k8sutil.NewErrorEvent("Pod creation failed", err, d.apiObject))
+			}
 
 		case <-time.After(inspectionInterval):
 			// Trigger inspection

pkg/deployment/pod_inspector.go

@@ -24,6 +24,7 @@ package deployment
 
 import (
 	"github.com/arangodb/k8s-operator/pkg/util/k8sutil"
+	"k8s.io/api/core/v1"
 
 	api "github.com/arangodb/k8s-operator/pkg/apis/arangodb/v1alpha"
 	"github.com/arangodb/k8s-operator/pkg/metrics"
@@ -37,9 +38,8 @@ var (
 // the member status of the deployment accordingly.
 func (d *Deployment) inspectPods() error {
 	log := d.deps.Log
+	var events []*v1.Event
 
-	log.Debug().Msg("inspecting pods")
-	defer log.Debug().Msg("inspected pods")
 	pods, err := d.deps.KubeCli.CoreV1().Pods(d.apiObject.GetNamespace()).List(k8sutil.DeploymentListOpt(d.apiObject.GetName()))
 	if err != nil {
 		log.Debug().Err(err).Msg("Failed to list pods")
@@ -47,7 +47,6 @@ func (d *Deployment) inspectPods() error {
 	}
 
 	// Update member status from all pods found
-	log.Debug().Int("pods", len(pods.Items)).Msg("iterating over pods")
 	for _, p := range pods.Items {
 		// Check ownership
 		if !d.isOwnerOf(&p) {
@@ -66,13 +65,13 @@ func (d *Deployment) inspectPods() error {
 		}
 
 		// Update state
-		log.Debug().Str("pod-name", p.GetName()).Msg("found member status for pod")
 		var readyStatus bool
 		reason := "Pod Not Ready"
 		if readyStatus = k8sutil.IsPodReady(&p); readyStatus {
 			reason = "Pod Ready"
 		}
 		if memberStatus.Conditions.Update(api.ConditionTypeReady, readyStatus, reason, "") {
+			log.Debug().Str("pod-name", p.GetName()).Msg("Updated member status member for pod")
 			if err := d.status.Members.UpdateMemberStatus(memberStatus, group); err != nil {
 				return maskAny(err)
 			}
@@ -92,7 +91,10 @@ func (d *Deployment) inspectPods() error {
 	d.status.Members.ForeachServerGroup(func(group api.ServerGroup, members *api.MemberStatusList) error {
 		for _, m := range *members {
 			if podName := m.PodName; podName != "" {
-				if !podExists(podName) {
+				if !podExists(podName) && m.State != api.MemberStateNone {
+					m.State = api.MemberStateNone // This is trigger a recreate of the pod.
+					// Create event
+					events = append(events, k8sutil.NewPodGoneEvent(podName, group.AsRole(), d.apiObject))
 					if m.Conditions.Update(api.ConditionTypeReady, false, "Pod Does Not Exist", "") {
 						if err := d.status.Members.UpdateMemberStatus(m, group); err != nil {
 							return maskAny(err)
@@ -117,5 +119,10 @@ func (d *Deployment) inspectPods() error {
 	if err := d.updateCRStatus(); err != nil {
 		return maskAny(err)
 	}
+
+	// Create events
+	for _, evt := range events {
+		d.createEvent(evt)
+	}
 	return nil
 }

pkg/util/k8sutil/events.go

@@ -59,6 +59,15 @@ func NewMemberRemoveEvent(memberName, role string, apiObject APIObject) *v1.Even
 	return event
 }
 
+// NewPodGoneEvent creates an event indicating that a pod is missing
+func NewPodGoneEvent(podName, role string, apiObject APIObject) *v1.Event {
+	event := newDeploymentEvent(apiObject)
+	event.Type = v1.EventTypeNormal
+	event.Reason = fmt.Sprintf("Pod Of %s Gone", strings.Title(role))
+	event.Message = fmt.Sprintf("Pod %s of member %s is gone", podName, role)
+	return event
+}
+
 // NewErrorEvent creates an even of type error.
 func NewErrorEvent(reason string, err error, apiObject APIObject) *v1.Event {
 	event := newDeploymentEvent(apiObject)